From ca3cdc971ec104ecedbe73cc9ff7641c842c54c3 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Fri, 26 Aug 2022 16:26:25 +0200 Subject: [PATCH 01/45] [REWORK] Global singleton implements context. --- src/main/java/io/github/s7i/doer/Context.java | 35 ++++++++++--------- src/main/java/io/github/s7i/doer/Globals.java | 9 ++--- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index 95113fb..4fadab3 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -1,28 +1,25 @@ package io.github.s7i.doer; -import static io.github.s7i.doer.Doer.console; -import static java.util.Objects.nonNull; -import static java.util.Objects.requireNonNull; - import io.github.s7i.doer.domain.kafka.output.KafkaOutputCreator; import io.github.s7i.doer.domain.kafka.output.KafkaUri; -import io.github.s7i.doer.domain.output.Output; -import io.github.s7i.doer.domain.output.OutputFactory; -import io.github.s7i.doer.domain.output.OutputKind; -import io.github.s7i.doer.domain.output.OutputProvider; -import io.github.s7i.doer.domain.output.UriResolver; +import io.github.s7i.doer.domain.output.*; import io.github.s7i.doer.domain.output.creator.FileOutputCreator; import io.github.s7i.doer.domain.output.creator.HttpOutputCreator; import io.github.s7i.doer.util.QuitWatcher; +import lombok.Builder; +import lombok.Builder.Default; +import lombok.Getter; +import org.slf4j.LoggerFactory; + import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.Map; import java.util.function.Predicate; -import lombok.Builder; -import lombok.Builder.Default; -import lombok.Getter; -import org.slf4j.LoggerFactory; + +import static io.github.s7i.doer.Doer.console; +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; public interface Context { @@ -49,9 +46,13 @@ public Initializer(InitialParameters parameters) { new QuitWatcher().watchForQuit(() -> System.exit(Doer.EC_QUIT)); } + public Context context() { + return Globals.INSTANCE; + } + private static void shutdown() { console().info("Init shutdown procedure..."); - Globals.INSTANCE.stopHooks.stream().forEach(Runnable::run); + Globals.INSTANCE.stopHooks.forEach(Runnable::run); console().info("Shutdown completed."); } } @@ -81,7 +82,7 @@ default Output buildOutput(OutputProvider outputProvider) { factory.register(OutputKind.KAFKA, kafka); return factory.resolve(new UriResolver(outputProvider.getOutput())) - .orElseThrow(); + .orElseThrow(); } default Map getParams() { @@ -99,8 +100,8 @@ default boolean hasFlag(String flag) { final var split = flags.split("\\,"); final var hasFlag = Arrays.stream(split) - .filter(Predicate.not(String::isBlank)) - .anyMatch(flag::equals); + .filter(Predicate.not(String::isBlank)) + .anyMatch(flag::equals); if (hasFlag) { LoggerFactory.getLogger(Context.class).debug("ON FLAG: {}", flag); diff --git a/src/main/java/io/github/s7i/doer/Globals.java b/src/main/java/io/github/s7i/doer/Globals.java index 790c7b4..d6a99ac 100644 --- a/src/main/java/io/github/s7i/doer/Globals.java +++ b/src/main/java/io/github/s7i/doer/Globals.java @@ -2,18 +2,19 @@ import io.github.s7i.doer.domain.kafka.KafkaFactory; import io.github.s7i.doer.domain.output.OutputFactory; +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + import java.nio.file.Path; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; @Slf4j -public enum Globals { +public enum Globals implements Context { INSTANCE; private final Map scopeMap = new ConcurrentHashMap<>(); From 9ae895a2c549736fd775a31ad7fbd87bbb2de714 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Fri, 26 Aug 2022 16:37:24 +0200 Subject: [PATCH 02/45] [ADD] record ingest draft, output to console --- src/main/java/io/github/s7i/doer/Doer.java | 5 ++- .../io/github/s7i/doer/command/Ingest.java | 42 +++++++++++++++++++ .../doer/domain/ingest/IngestProcessor.java | 28 +++++++++++++ .../manifest/ingest/IngestRecordManifest.java | 28 +++++++++++++ .../ingest/record/IngestRecordTest.groovy | 26 ++++++++++++ src/test/resources/ingest/ingest-record.yml | 7 ++++ 6 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/github/s7i/doer/command/Ingest.java create mode 100644 src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java create mode 100644 src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java create mode 100644 src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy create mode 100644 src/test/resources/ingest/ingest-record.yml diff --git a/src/main/java/io/github/s7i/doer/Doer.java b/src/main/java/io/github/s7i/doer/Doer.java index 6f62309..d31f4c4 100644 --- a/src/main/java/io/github/s7i/doer/Doer.java +++ b/src/main/java/io/github/s7i/doer/Doer.java @@ -1,5 +1,6 @@ package io.github.s7i.doer; +import io.github.s7i.doer.command.Ingest; import io.github.s7i.doer.command.KafkaFeeder; import io.github.s7i.doer.command.ProtoProcessor; import io.github.s7i.doer.command.Rocks; @@ -19,7 +20,9 @@ KafkaDump.class, ProtoProcessor.class, Rocks.class, - ReplaceInFile.class}) + ReplaceInFile.class, + Ingest.class +}) public class Doer implements Runnable { static final Logger CONSOLE = LoggerFactory.getLogger("doer.console"); diff --git a/src/main/java/io/github/s7i/doer/command/Ingest.java b/src/main/java/io/github/s7i/doer/command/Ingest.java new file mode 100644 index 0000000..3f1f030 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/command/Ingest.java @@ -0,0 +1,42 @@ +package io.github.s7i.doer.command; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.domain.ingest.IngestProcessor; +import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +import java.io.File; +import java.util.concurrent.Callable; + +@Command(name = "ingest") +public class Ingest implements Callable, YamlParser { + + @Option(names = {"-y"}, required = true) + File yaml; + + @Override + public File getYamlFile() { + if (!yaml.exists()) { + throw new IllegalStateException("The manifest file doesn't exists: " + yaml); + } + return yaml; + } + + + @Override + public Integer call() throws Exception { + + var manifest = parseYaml(IngestRecordManifest.class); + + var ctx = new Context.Initializer(Context.InitialParameters.builder() + .workDir(yaml.toPath().toAbsolutePath().getParent()) + .params(manifest.getParams()) + .build()) + .context(); + + new IngestProcessor(ctx).process(manifest); + + return 0; + } +} diff --git a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java new file mode 100644 index 0000000..7e71aae --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java @@ -0,0 +1,28 @@ +package io.github.s7i.doer.domain.ingest; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.domain.output.ConsoleOutput; +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; +import io.github.s7i.doer.util.PropertyResolver; +import lombok.RequiredArgsConstructor; + +import java.nio.charset.StandardCharsets; + +@RequiredArgsConstructor +public class IngestProcessor { + + private final Context context; + private Output output = new ConsoleOutput(); + + public void process(IngestRecordManifest manifest) { + + var propertyResolver = new PropertyResolver(context.getParams()); + + manifest.getRecords().stream() + .map(rec -> Output.Load.builder() + .data(propertyResolver.resolve(rec.getRecord()).getBytes(StandardCharsets.UTF_8)) + .build()) + .forEach(output::emit); + } +} diff --git a/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java b/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java new file mode 100644 index 0000000..f41e6e9 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java @@ -0,0 +1,28 @@ +package io.github.s7i.doer.manifest.ingest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.github.s7i.doer.config.Base; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.List; + +@Getter +public class IngestRecordManifest extends Base { + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class Record { + String record; + String key; + } + + @JsonProperty("ingest") + List records = Collections.emptyList(); + + +} diff --git a/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy new file mode 100644 index 0000000..e85770c --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy @@ -0,0 +1,26 @@ +package io.github.s7i.doer.ingest.record + +import io.github.s7i.doer.Globals +import io.github.s7i.doer.command.YamlParser +import io.github.s7i.doer.domain.ingest.IngestProcessor +import io.github.s7i.doer.manifest.ingest.IngestRecordManifest +import spock.lang.Specification + +import java.nio.file.Path + +class IngestRecordTest extends Specification { + + def "Parse Manifest" () { + + given: + YamlParser yml = { Path.of("src/test/resources/ingest/ingest-record.yml").toFile() } + def manifest = yml.parseYaml(IngestRecordManifest.class) + + new IngestProcessor(Globals.INSTANCE).process(manifest) + + expect: + manifest.getKind() == 'ingest' + + } + +} diff --git a/src/test/resources/ingest/ingest-record.yml b/src/test/resources/ingest/ingest-record.yml new file mode 100644 index 0000000..ccca702 --- /dev/null +++ b/src/test/resources/ingest/ingest-record.yml @@ -0,0 +1,7 @@ +version: v1 +kind: ingest +ingest: + - record: record 1 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + - record: record 2 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + - record: record 3 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + key: key-record 3 From 10fef775dd00b750b3377245f035b007b6f7dbda Mon Sep 17 00:00:00 2001 From: nefro85 Date: Fri, 26 Aug 2022 20:13:00 +0200 Subject: [PATCH 03/45] [ADD] Pipeline as output kind, preparation for building pipes --- src/main/java/io/github/s7i/doer/Context.java | 3 +++ src/main/java/io/github/s7i/doer/Globals.java | 3 +++ .../s7i/doer/domain/output/OutputKind.java | 2 +- .../doer/domain/output/PipelineOutput.java | 27 +++++++++++++++++++ .../s7i/doer/domain/output/UriResolver.java | 3 +++ .../output/creator/PipelineOutputCreator.java | 16 +++++++++++ .../io/github/s7i/doer/pipeline/LoadPipe.java | 10 +++++++ .../io/github/s7i/doer/pipeline/Pipeline.java | 9 +++++++ 8 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java create mode 100644 src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java create mode 100644 src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java create mode 100644 src/main/java/io/github/s7i/doer/pipeline/Pipeline.java diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index 4fadab3..1d8de90 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -5,6 +5,7 @@ import io.github.s7i.doer.domain.output.*; import io.github.s7i.doer.domain.output.creator.FileOutputCreator; import io.github.s7i.doer.domain.output.creator.HttpOutputCreator; +import io.github.s7i.doer.domain.output.creator.PipelineOutputCreator; import io.github.s7i.doer.util.QuitWatcher; import lombok.Builder; import lombok.Builder.Default; @@ -75,11 +76,13 @@ default Output buildOutput(OutputProvider outputProvider) { FileOutputCreator foc = () -> getBaseDir().resolve(outputProvider.getOutput()); HttpOutputCreator http = outputProvider::getOutput; KafkaOutputCreator kafka = new KafkaUri(outputProvider, this); + PipelineOutputCreator pipeline = () -> Globals.INSTANCE.pipeline.connect(outputProvider.getOutput()); final var factory = getOutputFactory(); factory.register(OutputKind.FILE, foc); factory.register(OutputKind.HTTP, http); factory.register(OutputKind.KAFKA, kafka); + factory.register(OutputKind.PIPELINE, pipeline); return factory.resolve(new UriResolver(outputProvider.getOutput())) .orElseThrow(); diff --git a/src/main/java/io/github/s7i/doer/Globals.java b/src/main/java/io/github/s7i/doer/Globals.java index d6a99ac..3aeb20b 100644 --- a/src/main/java/io/github/s7i/doer/Globals.java +++ b/src/main/java/io/github/s7i/doer/Globals.java @@ -2,6 +2,7 @@ import io.github.s7i.doer.domain.kafka.KafkaFactory; import io.github.s7i.doer.domain.output.OutputFactory; +import io.github.s7i.doer.pipeline.Pipeline; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -22,6 +23,8 @@ public enum Globals implements Context { List stopHooks = new ArrayList<>(); @Getter KafkaFactory kafka = new KafkaFactory(); + @Getter + Pipeline pipeline = new Pipeline(); public Scope getScope() { diff --git a/src/main/java/io/github/s7i/doer/domain/output/OutputKind.java b/src/main/java/io/github/s7i/doer/domain/output/OutputKind.java index fc354af..1c1c735 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/OutputKind.java +++ b/src/main/java/io/github/s7i/doer/domain/output/OutputKind.java @@ -2,5 +2,5 @@ public enum OutputKind { - CONSOLE, FILE, KAFKA, HTTP + CONSOLE, FILE, KAFKA, HTTP, PIPELINE } diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java new file mode 100644 index 0000000..08a4ca5 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -0,0 +1,27 @@ +package io.github.s7i.doer.domain.output; + +import io.github.s7i.doer.pipeline.LoadPipe; +import lombok.RequiredArgsConstructor; + + +@RequiredArgsConstructor +public class PipelineOutput implements Output { + + + private final LoadPipe loadPipe; + + @Override + public void open() { + + } + + @Override + public void emit(Load load) { + loadPipe.push(load); + } + + @Override + public void close() throws Exception { + + } +} diff --git a/src/main/java/io/github/s7i/doer/domain/output/UriResolver.java b/src/main/java/io/github/s7i/doer/domain/output/UriResolver.java index 06e77fa..4a32847 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/UriResolver.java +++ b/src/main/java/io/github/s7i/doer/domain/output/UriResolver.java @@ -1,6 +1,7 @@ package io.github.s7i.doer.domain.output; import io.github.s7i.doer.DoerException; + import java.net.URI; import java.net.URISyntaxException; @@ -28,6 +29,8 @@ public OutputKind resolveOutputKind() { return OutputKind.valueOf(authority.toUpperCase()); case "kafka": return OutputKind.KAFKA; + case "pipeline": + return OutputKind.PIPELINE; default: return OutputKind.FILE; } diff --git a/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java new file mode 100644 index 0000000..b38caf2 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java @@ -0,0 +1,16 @@ +package io.github.s7i.doer.domain.output.creator; + +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.domain.output.PipelineOutput; +import io.github.s7i.doer.pipeline.LoadPipe; + +public interface PipelineOutputCreator extends OutputCreator { + + + LoadPipe getLoadPipe(); + + @Override + default Output create() { + return new PipelineOutput(getLoadPipe()); + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java b/src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java new file mode 100644 index 0000000..a662787 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java @@ -0,0 +1,10 @@ +package io.github.s7i.doer.pipeline; + +import io.github.s7i.doer.domain.output.Output.Load; + +public interface LoadPipe { + + void push(Load load); + + Load pull(); +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java new file mode 100644 index 0000000..5f2aae7 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -0,0 +1,9 @@ +package io.github.s7i.doer.pipeline; + +public class Pipeline { + + public LoadPipe connect(String name) { + return null; + } + +} From cfce2b1129d74a479231a9a6abd468e7534f3055 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 27 Aug 2022 10:16:36 +0200 Subject: [PATCH 04/45] [ADD] handler to run as pipeline element --- .../java/io/github/s7i/doer/command/Ingest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/io/github/s7i/doer/command/Ingest.java b/src/main/java/io/github/s7i/doer/command/Ingest.java index 3f1f030..16f9c5f 100644 --- a/src/main/java/io/github/s7i/doer/command/Ingest.java +++ b/src/main/java/io/github/s7i/doer/command/Ingest.java @@ -1,6 +1,7 @@ package io.github.s7i.doer.command; import io.github.s7i.doer.Context; +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.ingest.IngestProcessor; import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; import picocli.CommandLine.Command; @@ -12,6 +13,18 @@ @Command(name = "ingest") public class Ingest implements Callable, YamlParser { + public static Runnable createCommandInstance(File yaml) { + return () -> { + var ingest = new Ingest(); + ingest.yaml = yaml; + try { + ingest.call(); + } catch (Exception e) { + throw new DoerException(e); + } + }; + } + @Option(names = {"-y"}, required = true) File yaml; From 1b1e82d6312eb2d5ba57ca32de541a8cf9f0c904 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 27 Aug 2022 10:35:48 +0200 Subject: [PATCH 05/45] [ADD] 'ingest' and 'config' manifest support --- .../doer/command/util/CommandManifest.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 76a77c0..39d4b46 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -1,7 +1,15 @@ package io.github.s7i.doer.command.util; +import io.github.s7i.doer.command.Ingest; import io.github.s7i.doer.command.KafkaFeeder; import io.github.s7i.doer.command.dump.KafkaDump; +import io.github.s7i.doer.domain.ConfigProcessor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -10,11 +18,6 @@ import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; @Command(name = "command-manifest", description = "Parsing command manifest yaml file") @Slf4j @@ -58,7 +61,7 @@ public void run() { try (var br = Files.newBufferedReader(yaml.toPath())) { String version = br.readLine(); String kind = br.readLine(); - kind = kind.substring(kind.lastIndexOf(':') + 1).trim(); + kind = kind.substring(kind.lastIndexOf(':') + 1).trim().toLowerCase(); switch (kind) { case "kafka-ingest": @@ -67,6 +70,12 @@ public void run() { case "kafka-dump": pool.execute(new TaskWrapper(kind, KafkaDump.createCommandInstance(yaml))); break; + case "ingest": + pool.execute(new TaskWrapper(kind, Ingest.createCommandInstance(yaml))); + break; + case "config": + pool.execute(new TaskWrapper(kind, () -> new ConfigProcessor(yaml).processConfig())); + break; default: log.warn("unsupported kind/type of file: {}", kind); break; From 4e4aa4a3c248e2e9ca13f85a82921ef3a8ca13bf Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 27 Aug 2022 10:37:07 +0200 Subject: [PATCH 06/45] [ADD] pipeline backend initialization process --- src/main/java/io/github/s7i/doer/Context.java | 2 ++ .../s7i/doer/domain/ConfigProcessor.java | 28 ++++++++++++++++++ .../s7i/doer/manifest/ConfigManifest.java | 6 ++++ .../io/github/s7i/doer/pipeline/Pipeline.java | 23 +++++++++++++++ .../s7i/doer/pipeline/PipelineTest.groovy | 29 +++++++++++++++++++ .../resources/pipeline/pipeline-config.yml | 7 +++++ 6 files changed, 95 insertions(+) create mode 100644 src/main/java/io/github/s7i/doer/domain/ConfigProcessor.java create mode 100644 src/main/java/io/github/s7i/doer/manifest/ConfigManifest.java create mode 100644 src/test/groovy/io/github/s7i/doer/pipeline/PipelineTest.groovy create mode 100644 src/test/resources/pipeline/pipeline-config.yml diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index 1d8de90..7615ce3 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -6,6 +6,7 @@ import io.github.s7i.doer.domain.output.creator.FileOutputCreator; import io.github.s7i.doer.domain.output.creator.HttpOutputCreator; import io.github.s7i.doer.domain.output.creator.PipelineOutputCreator; +import io.github.s7i.doer.pipeline.Pipeline; import io.github.s7i.doer.util.QuitWatcher; import lombok.Builder; import lombok.Builder.Default; @@ -40,6 +41,7 @@ class Initializer { } public Initializer(InitialParameters parameters) { + Pipeline.initFrom(parameters::getParams); var scope = Globals.INSTANCE.getScope(); scope.setRoot(parameters::getWorkDir); scope.setParams(parameters::getParams); diff --git a/src/main/java/io/github/s7i/doer/domain/ConfigProcessor.java b/src/main/java/io/github/s7i/doer/domain/ConfigProcessor.java new file mode 100644 index 0000000..1625529 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/ConfigProcessor.java @@ -0,0 +1,28 @@ +package io.github.s7i.doer.domain; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.command.YamlParser; +import io.github.s7i.doer.manifest.ConfigManifest; +import lombok.RequiredArgsConstructor; + +import java.io.File; + +@RequiredArgsConstructor +public class ConfigProcessor implements YamlParser { + + private final File yaml; + + @Override + public File getYamlFile() { + return yaml; + } + + public void processConfig() { + var manifest = parseYaml(ConfigManifest.class); + new Context.Initializer(Context.InitialParameters.builder() + .workDir(yaml.toPath().toAbsolutePath().getParent()) + .params(manifest.getParams()) + .build()); + + } +} diff --git a/src/main/java/io/github/s7i/doer/manifest/ConfigManifest.java b/src/main/java/io/github/s7i/doer/manifest/ConfigManifest.java new file mode 100644 index 0000000..16b6a5d --- /dev/null +++ b/src/main/java/io/github/s7i/doer/manifest/ConfigManifest.java @@ -0,0 +1,6 @@ +package io.github.s7i.doer.manifest; + +import io.github.s7i.doer.config.Base; + +public class ConfigManifest extends Base { +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 5f2aae7..d96f0b7 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -1,7 +1,30 @@ package io.github.s7i.doer.pipeline; +import io.github.s7i.doer.Globals; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +@Slf4j public class Pipeline { + public static final String DOER_PIPELINE = "doer.pipeline.backend"; + + public static void initFrom(Supplier> params) { + var p = params.get().entrySet() + .stream() + .filter(es -> es.getKey().startsWith(DOER_PIPELINE)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + Globals.INSTANCE.getPipeline().init(p); + } + + + void init(Map params) { + + } + public LoadPipe connect(String name) { return null; } diff --git a/src/test/groovy/io/github/s7i/doer/pipeline/PipelineTest.groovy b/src/test/groovy/io/github/s7i/doer/pipeline/PipelineTest.groovy new file mode 100644 index 0000000..1308f16 --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/pipeline/PipelineTest.groovy @@ -0,0 +1,29 @@ +package io.github.s7i.doer.pipeline + +import io.github.s7i.doer.Globals +import io.github.s7i.doer.domain.ConfigProcessor +import spock.lang.Specification + +import java.nio.file.Path + +class PipelineTest extends Specification { + + def "should init pipeline"() { + given: + def params = [ + "doer.pipeline.backend" : "kafka", + "doer.pipeline.backend.kafka.properties": "bootstrap.servers=kafka:9093\n" + + ] + def pipeline = Mock(Pipeline) { + } + Globals.INSTANCE.pipeline = pipeline + + when: + new ConfigProcessor(Path.of("src/test/resources/pipeline/pipeline-config.yml").toFile()).processConfig() + + then: + 1 * pipeline.init(params) + + } +} diff --git a/src/test/resources/pipeline/pipeline-config.yml b/src/test/resources/pipeline/pipeline-config.yml new file mode 100644 index 0000000..5d41b73 --- /dev/null +++ b/src/test/resources/pipeline/pipeline-config.yml @@ -0,0 +1,7 @@ +version: v1 +kind: Config +params: + doer.pipeline.backend: kafka + doer.pipeline.backend.kafka.properties: |+ + bootstrap.servers=kafka:9093 + noop.param: random From 6d9f33d872b6f93202a2f89b1a9314041e5a6da1 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Fri, 5 Nov 2021 20:01:09 +0100 Subject: [PATCH 07/45] [REWORK] flag extractor / cherry-pick 9692db3b64497c45dee5314e7612763194e8ab54 --- src/main/java/io/github/s7i/doer/Context.java | 29 +-------------- .../io/github/s7i/doer/manifest/Manifest.java | 4 +- .../s7i/doer/util/ParamFlagExtractor.java | 37 +++++++++++++++++++ 3 files changed, 42 insertions(+), 28 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/util/ParamFlagExtractor.java diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index 7615ce3..bcb23e1 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -7,23 +7,20 @@ import io.github.s7i.doer.domain.output.creator.HttpOutputCreator; import io.github.s7i.doer.domain.output.creator.PipelineOutputCreator; import io.github.s7i.doer.pipeline.Pipeline; +import io.github.s7i.doer.util.ParamFlagExtractor; import io.github.s7i.doer.util.QuitWatcher; import lombok.Builder; import lombok.Builder.Default; import lombok.Getter; -import org.slf4j.LoggerFactory; import java.nio.file.Path; -import java.util.Arrays; import java.util.Collections; import java.util.Map; -import java.util.function.Predicate; import static io.github.s7i.doer.Doer.console; -import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; -public interface Context { +public interface Context extends ParamFlagExtractor { @Builder @Getter @@ -93,26 +90,4 @@ default Output buildOutput(OutputProvider outputProvider) { default Map getParams() { return Globals.INSTANCE.getScope().getParams().get(); } - - default boolean hasFlag(String flag) { - final var flags = getParams().get(Doer.FLAGS); - - if (nonNull(flags)) { - if (flags.equals(flag)) { - LoggerFactory.getLogger(Context.class).debug("ON FLAG: {}", flag); - return true; - } - - final var split = flags.split("\\,"); - final var hasFlag = Arrays.stream(split) - .filter(Predicate.not(String::isBlank)) - .anyMatch(flag::equals); - - if (hasFlag) { - LoggerFactory.getLogger(Context.class).debug("ON FLAG: {}", flag); - } - return hasFlag; - } - return false; - } } diff --git a/src/main/java/io/github/s7i/doer/manifest/Manifest.java b/src/main/java/io/github/s7i/doer/manifest/Manifest.java index c8545aa..9166c40 100644 --- a/src/main/java/io/github/s7i/doer/manifest/Manifest.java +++ b/src/main/java/io/github/s7i/doer/manifest/Manifest.java @@ -1,8 +1,10 @@ package io.github.s7i.doer.manifest; +import io.github.s7i.doer.util.ParamFlagExtractor; + import java.util.Map; -public interface Manifest { +public interface Manifest extends ParamFlagExtractor { T getSpecification(); diff --git a/src/main/java/io/github/s7i/doer/util/ParamFlagExtractor.java b/src/main/java/io/github/s7i/doer/util/ParamFlagExtractor.java new file mode 100644 index 0000000..77f9b76 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/util/ParamFlagExtractor.java @@ -0,0 +1,37 @@ +package io.github.s7i.doer.util; + +import io.github.s7i.doer.Doer; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Map; +import java.util.function.Predicate; + +import static java.util.Objects.nonNull; + +public interface ParamFlagExtractor { + + Map getParams(); + + default boolean hasFlag(String flag) { + final var flags = getParams().get(Doer.FLAGS); + + if (nonNull(flags)) { + if (flags.equals(flag)) { + LoggerFactory.getLogger(Doer.class).debug("ON FLAG: {}", flag); + return true; + } + + final var split = flags.split("\\,"); + final var hasFlag = Arrays.stream(split) + .filter(Predicate.not(String::isBlank)) + .anyMatch(flag::equals); + + if (hasFlag) { + LoggerFactory.getLogger(Doer.class).debug("ON FLAG: {}", flag); + } + return hasFlag; + } + return false; + } +} From d68b087ca1e209332796b6e34c0f029eac03cb6e Mon Sep 17 00:00:00 2001 From: Mariusz Sygnowski Date: Sat, 27 Aug 2022 13:55:38 +0200 Subject: [PATCH 08/45] [ADD] source (ingest) and sink with pipeline connection draft --- src/test/resources/pipeline/ingest.yml | 7 +++++++ src/test/resources/pipeline/sink.yml | 7 +++++++ 2 files changed, 14 insertions(+) create mode 100644 src/test/resources/pipeline/ingest.yml create mode 100644 src/test/resources/pipeline/sink.yml diff --git a/src/test/resources/pipeline/ingest.yml b/src/test/resources/pipeline/ingest.yml new file mode 100644 index 0000000..723488e --- /dev/null +++ b/src/test/resources/pipeline/ingest.yml @@ -0,0 +1,7 @@ +version: v1 +kind: ingest +params: + doer.pipeline.bind: from-ingest-to-sink + doer.pipeline.id: id-ingest1 +spec: + - record: my first record diff --git a/src/test/resources/pipeline/sink.yml b/src/test/resources/pipeline/sink.yml new file mode 100644 index 0000000..a31fe85 --- /dev/null +++ b/src/test/resources/pipeline/sink.yml @@ -0,0 +1,7 @@ +version: v1 +kind: sink +params: + doer.pipeline.bind: from-ingest-to-sink + doer.pipeline.id: id-sink1 +spec: + - output: doer://console From 5ecc823a9bd30a7a37c7987847faba23f86d37cd Mon Sep 17 00:00:00 2001 From: Mariusz Sygnowski Date: Sat, 27 Aug 2022 13:58:17 +0200 Subject: [PATCH 09/45] [REWORK] normalisation - spec field as alias of ingest definition --- .../github/s7i/doer/manifest/ingest/IngestRecordManifest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java b/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java index f41e6e9..b8d80e9 100644 --- a/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java +++ b/src/main/java/io/github/s7i/doer/manifest/ingest/IngestRecordManifest.java @@ -1,5 +1,6 @@ package io.github.s7i.doer.manifest.ingest; +import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonProperty; import io.github.s7i.doer.config.Base; import lombok.AllArgsConstructor; @@ -22,6 +23,7 @@ public static class Record { } @JsonProperty("ingest") + @JsonAlias("spec") List records = Collections.emptyList(); From 6fb21ce7d6d51fcd3c956d9557e5563eccd961f9 Mon Sep 17 00:00:00 2001 From: Mariusz Sygnowski Date: Sat, 27 Aug 2022 14:30:44 +0200 Subject: [PATCH 10/45] [REWORK] concept draft update --- spec/manifest/pipeline.yml | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/spec/manifest/pipeline.yml b/spec/manifest/pipeline.yml index d4b41e5..1f345b2 100644 --- a/spec/manifest/pipeline.yml +++ b/spec/manifest/pipeline.yml @@ -1,13 +1,33 @@ # Pipeline manifest draft -version: "0.1" +version: v1 kind: pipeline -pipeline: a | b +params: + doer.pipeline.backend: kafka + doer.pipeline.backend.kafka.properties: |+ + bootstrap.servers=kafka:9093 spec: + - pipeline: a | b - name: a - description: "a" as instance of kafka-dump, making output to doer://pipeline - manifest: - file: kdump.yml + description: "a" as instance of source + manifest-file: source.yml - name: b - description: "b" as instance of kafka-feed, read from a pipeline - manifest: - file: kfeed.yml \ No newline at end of file + description: "b" as instance of sink + manifest-file: sink.yml +--- + +version: v2 +kind: pipeline +params: + doer.pipeline.backend: kafka + doer.pipeline.backend.kafka.properties: |+ + bootstrap.servers=kafka:9093 +spec: + pipeline: + - flow: a | b + elements: + - name: a + description: "a" as instance of source + manifest-file: source.yml + - name: b + description: "b" as instance of sink + manifest-file: sink.yml From 8a4fd429bdcc31233f74816eae4fde96687c24f7 Mon Sep 17 00:00:00 2001 From: Mariusz Sygnowski Date: Sun, 28 Aug 2022 12:47:16 +0200 Subject: [PATCH 11/45] [REWORK] PipePusher draft --- .../io/github/s7i/doer/domain/output/PipelineOutput.java | 6 +++--- .../doer/domain/output/creator/PipelineOutputCreator.java | 4 ++-- .../doer/pipeline/{LoadPipe.java => PipeConnection.java} | 4 ++-- src/main/java/io/github/s7i/doer/pipeline/PipePusher.java | 8 ++++++++ src/main/java/io/github/s7i/doer/pipeline/Pipeline.java | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) rename src/main/java/io/github/s7i/doer/pipeline/{LoadPipe.java => PipeConnection.java} (62%) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/PipePusher.java diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java index 08a4ca5..ab95863 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -1,6 +1,6 @@ package io.github.s7i.doer.domain.output; -import io.github.s7i.doer.pipeline.LoadPipe; +import io.github.s7i.doer.pipeline.PipeConnection; import lombok.RequiredArgsConstructor; @@ -8,7 +8,7 @@ public class PipelineOutput implements Output { - private final LoadPipe loadPipe; + private final PipeConnection pipeConnection; @Override public void open() { @@ -17,7 +17,7 @@ public void open() { @Override public void emit(Load load) { - loadPipe.push(load); + pipeConnection.push(() -> load); } @Override diff --git a/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java index b38caf2..70f91a1 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java +++ b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java @@ -2,12 +2,12 @@ import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.domain.output.PipelineOutput; -import io.github.s7i.doer.pipeline.LoadPipe; +import io.github.s7i.doer.pipeline.PipeConnection; public interface PipelineOutputCreator extends OutputCreator { - LoadPipe getLoadPipe(); + PipeConnection getLoadPipe(); @Override default Output create() { diff --git a/src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java similarity index 62% rename from src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java rename to src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index a662787..ae65953 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/LoadPipe.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -2,9 +2,9 @@ import io.github.s7i.doer.domain.output.Output.Load; -public interface LoadPipe { +public interface PipeConnection { - void push(Load load); + void push(PipePusher pusher); Load pull(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java new file mode 100644 index 0000000..48605c0 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java @@ -0,0 +1,8 @@ +package io.github.s7i.doer.pipeline; + +import io.github.s7i.doer.domain.output.Output; + +public interface PipePusher { + + Output.Load onNextLoad(); +} \ No newline at end of file diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index d96f0b7..d6e45ed 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -25,7 +25,7 @@ void init(Map params) { } - public LoadPipe connect(String name) { + public PipeConnection connect(String name) { return null; } From 31b13cbc8ffc5ae8ec2328d412716b0f362122d9 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Mon, 5 Sep 2022 17:51:21 +0200 Subject: [PATCH 12/45] [REWORK] improvements... --- .../doer/domain/output/PipelineOutput.java | 10 ++++- .../s7i/doer/pipeline/BlockingPipePusher.java | 40 +++++++++++++++++++ .../s7i/doer/pipeline/PipeConnection.java | 4 +- .../github/s7i/doer/pipeline/PipePusher.java | 2 + .../domain/output/PipelineOutputTest.groovy | 40 +++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java create mode 100644 src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java index ab95863..f14e4dc 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -1,23 +1,29 @@ package io.github.s7i.doer.domain.output; +import io.github.s7i.doer.pipeline.BlockingPipePusher; import io.github.s7i.doer.pipeline.PipeConnection; import lombok.RequiredArgsConstructor; +import static java.util.Objects.requireNonNull; + @RequiredArgsConstructor public class PipelineOutput implements Output { private final PipeConnection pipeConnection; + private BlockingPipePusher pusher; @Override public void open() { - + pusher = new BlockingPipePusher(); + pipeConnection.registerPusher(pusher); } @Override public void emit(Load load) { - pipeConnection.push(() -> load); + requireNonNull(pusher); + pusher.offer(load); } @Override diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java new file mode 100644 index 0000000..e516759 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java @@ -0,0 +1,40 @@ +package io.github.s7i.doer.pipeline; + +import io.github.s7i.doer.domain.output.Output; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class BlockingPipePusher implements PipePusher { + + private final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); + + @Override + public Output.Load onNextLoad() { + + Output.Load load; + while ((load = queue.peek()) == null) { + try { + TimeUnit.MILLISECONDS.sleep(10); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + + return load; + } + + @Override + public void onAccept() { + queue.poll(); + + } + + + public void offer(Output.Load load) { + queue.offer(load); + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index ae65953..b468a87 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -4,7 +4,7 @@ public interface PipeConnection { - void push(PipePusher pusher); + void registerPusher(PipePusher pusher); - Load pull(); + void registerPuller(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java index 48605c0..c5164b7 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java @@ -5,4 +5,6 @@ public interface PipePusher { Output.Load onNextLoad(); + + void onAccept(); } \ No newline at end of file diff --git a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy new file mode 100644 index 0000000..75170fb --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy @@ -0,0 +1,40 @@ +package io.github.s7i.doer.domain.output + +import io.github.s7i.doer.pipeline.BlockingPipePusher +import io.github.s7i.doer.pipeline.PipeConnection +import spock.lang.Specification + +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +class PipelineOutputTest extends Specification { + + + def "Pipeline output test"() { + given: + def thrSrv = Executors.newFixedThreadPool(1) + def load = Output.Load.builder() + .data("test".getBytes()) + .build() + BlockingPipePusher pusher + def pipeConnection = Mock(PipeConnection) { + registerPusher(_) >> { args -> + pusher = args[0] + } + } + def out = new PipelineOutput(pipeConnection) + + when: + out.open() + thrSrv.submit({ out.emit(load) }) + def nextLoad = pusher.onNextLoad() + pusher.onAccept() + + then: + thrSrv.shutdown() + thrSrv.awaitTermination(1, TimeUnit.SECONDS) + + nextLoad == load + } + +} From 7f76060497b938b653093db681219c9e4d679856 Mon Sep 17 00:00:00 2001 From: nefro85 <1815506+nefro85@users.noreply.github.com> Date: Tue, 10 Jan 2023 13:21:59 +0100 Subject: [PATCH 13/45] [ADD] gRPC service draft (#27) --- proto-doer/src/main/proto/doer.proto | 13 ++++ .../s7i/doer/domain/ServiceEntrypoint.java | 73 +++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 src/main/java/io/github/s7i/doer/domain/ServiceEntrypoint.java diff --git a/proto-doer/src/main/proto/doer.proto b/proto-doer/src/main/proto/doer.proto index 78984f1..c381f8a 100644 --- a/proto-doer/src/main/proto/doer.proto +++ b/proto-doer/src/main/proto/doer.proto @@ -5,6 +5,7 @@ package io.github.s7i.doer.proto; option java_multiple_files = true; import "google/protobuf/wrappers.proto"; +import "google/protobuf/empty.proto"; message DoerVersion { string name = 1; @@ -17,3 +18,15 @@ message Record { google.protobuf.BytesValue data = 4; } +service DoerService { + + rpc version (google.protobuf.Empty) returns (Sone); + rpc exec(Sone) returns (Sone); + rpc pulse(google.protobuf.Empty) returns (stream Sone); + +} + +message Sone { + string s = 1; +} + diff --git a/src/main/java/io/github/s7i/doer/domain/ServiceEntrypoint.java b/src/main/java/io/github/s7i/doer/domain/ServiceEntrypoint.java new file mode 100644 index 0000000..0b24f4a --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/ServiceEntrypoint.java @@ -0,0 +1,73 @@ +package io.github.s7i.doer.domain; + +import com.google.protobuf.Empty; +import io.github.s7i.doer.proto.DoerServiceGrpc; +import io.github.s7i.doer.proto.Sone; +import io.github.s7i.doer.util.GitProps; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; + +import java.time.Instant; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +@Slf4j +public class ServiceEntrypoint extends DoerServiceGrpc.DoerServiceImplBase { + + static class Pulsar { + ExecutorService es = Executors.newFixedThreadPool(1); + List> receivers = new CopyOnWriteArrayList<>(); + + void begin() { + es.submit(this::pulse); + + } + + void end() { + es.shutdown(); + receivers.clear(); + } + + void makePulse(Consumer p) { + final var now = Instant.now(); + + p.accept(String.format("%d|%s", now.getEpochSecond() ,now)); + } + + void pulse() { + while (!es.isShutdown()) { + try { + receivers.forEach(this::makePulse); + TimeUnit.SECONDS.sleep(1); + } catch (InterruptedException e) { + log.error("oops", e); + Thread.currentThread().interrupt(); + } + } + } + + void add(Consumer receiver) { + receivers.add(receiver); + } + } + + @Override + public void version(Empty request, StreamObserver responseObserver) { + responseObserver.onNext(sone(new GitProps().toString())); + responseObserver.onCompleted(); + } + + + @Override + public void pulse(Empty request, StreamObserver responseObserver) { + super.pulse(request, responseObserver); + } + + public Sone sone(String s) { + return Sone.newBuilder().setS(s).build(); + } +} From f9aa6781056702b2ee31f4dbed440547215e5c9b Mon Sep 17 00:00:00 2001 From: nefro85 Date: Mon, 30 Jan 2023 22:00:14 +0100 Subject: [PATCH 14/45] [REWORK] gRPC Server common class. (#27) --- .../s7i/doer/domain/grpc/GrpcServer.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 src/main/java/io/github/s7i/doer/domain/grpc/GrpcServer.java diff --git a/src/main/java/io/github/s7i/doer/domain/grpc/GrpcServer.java b/src/main/java/io/github/s7i/doer/domain/grpc/GrpcServer.java new file mode 100644 index 0000000..74f983d --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/grpc/GrpcServer.java @@ -0,0 +1,54 @@ +package io.github.s7i.doer.domain.grpc; + +import io.grpc.BindableService; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.Server; +import io.grpc.protobuf.services.HealthStatusManager; +import io.grpc.protobuf.services.ProtoReflectionService; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@Slf4j(topic = "doer.console") +@RequiredArgsConstructor +public class GrpcServer { + + private final HealthStatusManager hsm = new HealthStatusManager(); + private Server server; + + private final int port; + private final BindableService service; + + public GrpcServer startServer() throws IOException { + + server = Grpc.newServerBuilderForPort(port, InsecureServerCredentials.create()) + .addService(service) + .addService(ProtoReflectionService.newInstance()) + .addService(hsm.getHealthService()) + .build() + .start(); + + log.info("Server Started: {}", server); + + Runtime.getRuntime().addShutdownHook(new Thread(this::stopServer, "grpc-server-shutdown-hook")); + + try { + server.awaitTermination(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("oops", e); + } + log.info("bye bye"); + + return this; + } + + @SneakyThrows + private void stopServer() { + server.shutdownNow().awaitTermination(5, TimeUnit.SECONDS); + } +} From 9211795c4a1b4a741cac2c56045618398ec80fae Mon Sep 17 00:00:00 2001 From: nefro85 Date: Mon, 30 Jan 2023 22:01:28 +0100 Subject: [PATCH 15/45] [REWORK] unification usage of grpc server (#27) --- src/main/java/io/github/s7i/doer/Doer.java | 14 ++++++++ .../s7i/doer/pipeline/PipelineService.java | 32 +++---------------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/Doer.java b/src/main/java/io/github/s7i/doer/Doer.java index 28735d6..7790f11 100644 --- a/src/main/java/io/github/s7i/doer/Doer.java +++ b/src/main/java/io/github/s7i/doer/Doer.java @@ -4,6 +4,8 @@ import io.github.s7i.doer.command.dump.KafkaDump; import io.github.s7i.doer.command.util.CommandManifest; import io.github.s7i.doer.command.util.Misc; +import io.github.s7i.doer.domain.ServiceEntrypoint; +import io.github.s7i.doer.domain.grpc.GrpcServer; import io.github.s7i.doer.util.Banner; import io.github.s7i.doer.util.GitProps; import org.slf4j.Logger; @@ -11,6 +13,7 @@ import picocli.CommandLine; import picocli.CommandLine.Command; +import java.io.IOException; import java.util.Arrays; @Command(name = "doer", description = "let's do big things...", subcommands = { @@ -42,6 +45,17 @@ public static Logger console() { @CommandLine.Option(names = {"-v", "--version"}) private boolean showVersion; + @Command(name = "srv-grpc") + public int service(int port) { + try { + new GrpcServer(port, new ServiceEntrypoint()).startServer(); + return 0; + } catch (IOException e) { + console().error("grpc-server", e); + return EC_ERROR; + } + } + @Override public void run() { printBanner(); diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index 4877068..f04b17e 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -1,13 +1,9 @@ package io.github.s7i.doer.pipeline; +import io.github.s7i.doer.domain.grpc.GrpcServer; import io.github.s7i.doer.pipeline.proto.PipelinePublishRequest; import io.github.s7i.doer.pipeline.proto.PipelinePublishResponse; import io.github.s7i.doer.pipeline.proto.PipelineServiceGrpc; -import io.grpc.Grpc; -import io.grpc.InsecureServerCredentials; -import io.grpc.Server; -import io.grpc.protobuf.services.HealthStatusManager; -import io.grpc.protobuf.services.ProtoReflectionService; import io.grpc.stub.StreamObserver; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -15,15 +11,12 @@ import picocli.CommandLine.Command; import picocli.CommandLine.Option; -import java.util.concurrent.TimeUnit; - @Command(name = "pipeline") @Slf4j(topic = "doer.console") public class PipelineService implements Runnable { @Option(names ="--port", defaultValue = "6565") private Integer port; - private Server server; public static void main(String[] args) { new CommandLine(PipelineService.class).execute(args); @@ -48,25 +41,8 @@ public void publish(PipelinePublishRequest request, StreamObserver Date: Tue, 31 Jan 2023 18:26:26 +0100 Subject: [PATCH 16/45] [ADD] Command to emit protobuf Doer::Record entity (#16) [ADD] verification script --- docs/pipeline/pipeline-record.sh | 61 ++++++++++++++++ .../s7i/doer/pipeline/PipelineService.java | 70 +++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100755 docs/pipeline/pipeline-record.sh diff --git a/docs/pipeline/pipeline-record.sh b/docs/pipeline/pipeline-record.sh new file mode 100755 index 0000000..64dc370 --- /dev/null +++ b/docs/pipeline/pipeline-record.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +PROTO_SET=../../proto-doer/build/descriptors/main.desc +set -e + +doer misc pipeline record --help + +print_color() { + local TEXT=$1 + local CLR_NAME=$2 + local RED="\e[31m" + local GREEN="\e[32m" + local ENDCOLOR="\e[0m" + local COLOR=${!CLR_NAME} + echo -e "${COLOR}${TEXT}${ENDCOLOR}" +} + +green() { + print_color $1 "GREEN" +} + +make_record() { + doer misc pipeline record \ + --meta test.data=true \ + --key my-key \ + --data "some test data" +} + +make_record_pl() { + doer misc pipeline record \ + --meta test.data=true \ + --key my-key \ + --data "some test data" \ + --pipeline-load +} + +green "[HEX:record]" +make_record | xxd + +# https://github.com/protocolbuffers/protoscope +green "[protoscope]" +make_record | protoscope + +green "[protoscope:print-field-names]" + +make_record | protoscope \ + -descriptor-set ${PROTO_SET} \ + -message-type io.github.s7i.doer.proto.Record \ + -print-field-names + +green "[HEX:pipeline-load]" +make_record_pl | xxd + +green "[protoscope:pipeline-load]" +make_record_pl | protoscope + +green "[protoscope:pipeline-load:print-field-names]" +make_record_pl | protoscope \ + -descriptor-set ${PROTO_SET} \ + -message-type io.github.s7i.doer.pipeline.proto.PipelineLoad \ + -print-field-names \ No newline at end of file diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index f04b17e..2fcfd0f 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -1,9 +1,15 @@ package io.github.s7i.doer.pipeline; +import com.google.protobuf.Any; +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.StringValue; import io.github.s7i.doer.domain.grpc.GrpcServer; +import io.github.s7i.doer.pipeline.proto.PipelineLoad; import io.github.s7i.doer.pipeline.proto.PipelinePublishRequest; import io.github.s7i.doer.pipeline.proto.PipelinePublishResponse; import io.github.s7i.doer.pipeline.proto.PipelineServiceGrpc; +import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -11,6 +17,11 @@ import picocli.CommandLine.Command; import picocli.CommandLine.Option; +import java.io.IOException; +import java.util.Map; + +import static java.util.Objects.nonNull; + @Command(name = "pipeline") @Slf4j(topic = "doer.console") public class PipelineService implements Runnable { @@ -37,6 +48,65 @@ public void publish(PipelinePublishRequest request, StreamObserver + * # proto-doer/src/main/proto/doer.proto:14 + * + * message Record { + * google.protobuf.StringValue resource = 1; + * google.protobuf.StringValue key = 2; + * map meta = 3; + * google.protobuf.BytesValue data = 4; + * } + * + */ + @Command(name = "record", description = "Emit protobuf Record entity.") + public void record( + @Option(names = "--resource") + String resource, + @Option(names = "--key") + String key, + @Option(names = "--meta") + Map meta, + @Option(names = "--data") + String data, + @Option(names = {"-pl", "--pipeline-load"}, description = "Embed Record in PipelineLoad::Load.") + boolean pipelineLoad, + @Option(names= {"-h", "--help"}, usageHelp = true) + boolean help) { + + var b = Record.newBuilder(); + + if (nonNull(resource)) { + b.setResource(StringValue.of(resource)); + } + + if (nonNull(key)) { + b.setKey(StringValue.of(key)); + } + + if (nonNull(meta)) { + b.putAllMeta(meta); + } + + if (nonNull(data)) { + b.setData(BytesValue.of(ByteString.copyFromUtf8(data))); + } + + try { + byte[] bytes = pipelineLoad + ? PipelineLoad.newBuilder() + .setLoad(Any.pack(b.build())) + .build() + .toByteArray() + : b.build().toByteArray(); + + System.out.write(bytes); + } catch (IOException e) { + log.error("new record", e); + } + } + @SneakyThrows @Override From 32649b28edbe9dd5f87be71894392bbc99e519a9 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Wed, 8 Feb 2023 21:45:03 +0100 Subject: [PATCH 17/45] [REWORK] (#14) - doc rework --- .../pipeline.yml => docs/pipeline/readme.md | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) rename spec/manifest/pipeline.yml => docs/pipeline/readme.md (92%) diff --git a/spec/manifest/pipeline.yml b/docs/pipeline/readme.md similarity index 92% rename from spec/manifest/pipeline.yml rename to docs/pipeline/readme.md index 1f345b2..dd85bb1 100644 --- a/spec/manifest/pipeline.yml +++ b/docs/pipeline/readme.md @@ -1,33 +1,39 @@ -# Pipeline manifest draft -version: v1 -kind: pipeline -params: - doer.pipeline.backend: kafka - doer.pipeline.backend.kafka.properties: |+ - bootstrap.servers=kafka:9093 -spec: - - pipeline: a | b - - name: a - description: "a" as instance of source - manifest-file: source.yml - - name: b - description: "b" as instance of sink - manifest-file: sink.yml ---- - -version: v2 -kind: pipeline -params: - doer.pipeline.backend: kafka - doer.pipeline.backend.kafka.properties: |+ - bootstrap.servers=kafka:9093 -spec: - pipeline: - - flow: a | b - elements: - - name: a - description: "a" as instance of source - manifest-file: source.yml - - name: b - description: "b" as instance of sink - manifest-file: sink.yml +# Pipelines + +Sample of pipeline specification: + +```yaml +# Pipeline manifest draft +version: v1 +kind: pipeline +params: + doer.pipeline.backend: kafka + doer.pipeline.backend.kafka.properties: |+ + bootstrap.servers=kafka:9093 +spec: + - pipeline: a | b + - name: a + description: "a" as instance of source + manifest-file: source.yml + - name: b + description: "b" as instance of sink + manifest-file: sink.yml +--- + +version: v2 +kind: pipeline +params: + doer.pipeline.backend: kafka + doer.pipeline.backend.kafka.properties: |+ + bootstrap.servers=kafka:9093 +spec: + pipeline: + - flow: a | b + elements: + - name: a + description: "a" as instance of source + manifest-file: source.yml + - name: b + description: "b" as instance of sink + manifest-file: sink.yml +``` \ No newline at end of file From 868d19731cd3fd06bdca5fff3e4026820da244fc Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 19 Feb 2023 15:01:12 +0100 Subject: [PATCH 18/45] [REWORK] PipePublisher improvements [ADD] multi thread testing [TAG] Pipelines #14 --- .../s7i/doer/pipeline/BlockingPipePusher.java | 28 +++++++- .../domain/output/PipelineOutputTest.groovy | 64 +++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java index e516759..258dabb 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java @@ -9,15 +9,21 @@ @Slf4j public class BlockingPipePusher implements PipePusher { + public static final int SLEEP_FOR_CHANGE = 10; private final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); + /** + * Blocking call. + * + * @return + */ @Override public Output.Load onNextLoad() { Output.Load load; while ((load = queue.peek()) == null) { try { - TimeUnit.MILLISECONDS.sleep(10); + TimeUnit.MILLISECONDS.sleep(SLEEP_FOR_CHANGE); } catch (InterruptedException e) { Thread.currentThread().interrupt(); break; @@ -30,11 +36,27 @@ public Output.Load onNextLoad() { @Override public void onAccept() { queue.poll(); - } + /** + * Blocking call. + * + * @param load + */ public void offer(Output.Load load) { - queue.offer(load); + boolean accepted; + do { + accepted = queue.offer(load); + if (!accepted) { + try { + TimeUnit.MILLISECONDS.sleep(SLEEP_FOR_CHANGE); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + while (!accepted); } } diff --git a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy index 75170fb..aacd28f 100644 --- a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy @@ -4,6 +4,7 @@ import io.github.s7i.doer.pipeline.BlockingPipePusher import io.github.s7i.doer.pipeline.PipeConnection import spock.lang.Specification +import java.util.concurrent.CountDownLatch import java.util.concurrent.Executors import java.util.concurrent.TimeUnit @@ -37,4 +38,67 @@ class PipelineOutputTest extends Specification { nextLoad == load } + def "Pipeline output test with backlog"() { + given: + def loadTransfer =[] + def thrCount=0 + def thrSrv = Executors.newFixedThreadPool(10, { r-> + def t = new Thread(r , "pusher-${++thrCount}") + t.setDaemon(true) + return t + }) + BlockingPipePusher pusher + def pipeConnection = Mock(PipeConnection) { + registerPusher(_) >> { args -> + pusher = args[0] + } + } + def out = new PipelineOutput(pipeConnection) + + when: + out.open() + + def cdlPush = new CountDownLatch(10) + + 10.times { + def task = { + println "Running task ${Thread.currentThread().getName()}" + + out.emit(Output.Load.builder() + .resource(Thread.currentThread().getName()) + .data("test".getBytes()) + .build()) + + cdlPush.countDown() + } + thrSrv.submit(task) + println "Task submitted (# ${it})" + } + + def cdlWorkDone = new CountDownLatch(10) + new Thread({ + + 10.times { + def nextLoad = pusher.onNextLoad() + pusher.onAccept() + + loadTransfer << nextLoad + cdlWorkDone.countDown() + + println "pull done for ${nextLoad.getResource()}" + + } + }, "puller").start() + + + then: + cdlPush.await(5, TimeUnit.SECONDS) + cdlWorkDone.await(5, TimeUnit.SECONDS) + + loadTransfer.size() == 10 + + thrSrv.shutdown() + thrSrv.awaitTermination(5, TimeUnit.SECONDS) + } + } From e8a09ebbff8e8e36ddc9b16a8c8e6552b524410f Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 19 Feb 2023 15:33:11 +0100 Subject: [PATCH 19/45] [REWORK] Name correction connected with responsibility. [TAG] Pipelines #14 --- .../doer/domain/output/PipelineOutput.java | 8 +++---- ...ipePusher.java => BlockingPipePuller.java} | 2 +- .../s7i/doer/pipeline/PipeConnection.java | 6 ++--- .../{PipePusher.java => PipePuller.java} | 2 +- .../io/github/s7i/doer/pipeline/Pipeline.java | 2 ++ .../domain/output/PipelineOutputTest.groovy | 22 +++++++++---------- 6 files changed, 21 insertions(+), 21 deletions(-) rename src/main/java/io/github/s7i/doer/pipeline/{BlockingPipePusher.java => BlockingPipePuller.java} (96%) rename src/main/java/io/github/s7i/doer/pipeline/{PipePusher.java => PipePuller.java} (82%) diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java index f14e4dc..edf0eb6 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -1,6 +1,6 @@ package io.github.s7i.doer.domain.output; -import io.github.s7i.doer.pipeline.BlockingPipePusher; +import io.github.s7i.doer.pipeline.BlockingPipePuller; import io.github.s7i.doer.pipeline.PipeConnection; import lombok.RequiredArgsConstructor; @@ -12,12 +12,12 @@ public class PipelineOutput implements Output { private final PipeConnection pipeConnection; - private BlockingPipePusher pusher; + private BlockingPipePuller pusher; @Override public void open() { - pusher = new BlockingPipePusher(); - pipeConnection.registerPusher(pusher); + pusher = new BlockingPipePuller(); + pipeConnection.registerPuller(pusher); } @Override diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java similarity index 96% rename from src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java rename to src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java index 258dabb..c061ddc 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePusher.java +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java @@ -7,7 +7,7 @@ import java.util.concurrent.TimeUnit; @Slf4j -public class BlockingPipePusher implements PipePusher { +public class BlockingPipePuller implements PipePuller { public static final int SLEEP_FOR_CHANGE = 10; private final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index b468a87..5148445 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -1,10 +1,8 @@ package io.github.s7i.doer.pipeline; -import io.github.s7i.doer.domain.output.Output.Load; - public interface PipeConnection { - void registerPusher(PipePusher pusher); + void registerPuller(PipePuller puller); - void registerPuller(); + //void registerPuller(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/PipePuller.java similarity index 82% rename from src/main/java/io/github/s7i/doer/pipeline/PipePusher.java rename to src/main/java/io/github/s7i/doer/pipeline/PipePuller.java index c5164b7..57dca56 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipePuller.java @@ -2,7 +2,7 @@ import io.github.s7i.doer.domain.output.Output; -public interface PipePusher { +public interface PipePuller { Output.Load onNextLoad(); diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index d6e45ed..f82087c 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -1,6 +1,7 @@ package io.github.s7i.doer.pipeline; import io.github.s7i.doer.Globals; +import io.github.s7i.doer.util.Mark; import lombok.extern.slf4j.Slf4j; import java.util.Map; @@ -10,6 +11,7 @@ @Slf4j public class Pipeline { + @Mark.Param public static final String DOER_PIPELINE = "doer.pipeline.backend"; public static void initFrom(Supplier> params) { diff --git a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy index aacd28f..397d1bd 100644 --- a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy @@ -1,6 +1,6 @@ package io.github.s7i.doer.domain.output -import io.github.s7i.doer.pipeline.BlockingPipePusher +import io.github.s7i.doer.pipeline.BlockingPipePuller import io.github.s7i.doer.pipeline.PipeConnection import spock.lang.Specification @@ -17,10 +17,10 @@ class PipelineOutputTest extends Specification { def load = Output.Load.builder() .data("test".getBytes()) .build() - BlockingPipePusher pusher + BlockingPipePuller puller def pipeConnection = Mock(PipeConnection) { - registerPusher(_) >> { args -> - pusher = args[0] + registerPuller(_) >> { args -> + puller = args[0] } } def out = new PipelineOutput(pipeConnection) @@ -28,8 +28,8 @@ class PipelineOutputTest extends Specification { when: out.open() thrSrv.submit({ out.emit(load) }) - def nextLoad = pusher.onNextLoad() - pusher.onAccept() + def nextLoad = puller.onNextLoad() + puller.onAccept() then: thrSrv.shutdown() @@ -47,10 +47,10 @@ class PipelineOutputTest extends Specification { t.setDaemon(true) return t }) - BlockingPipePusher pusher + BlockingPipePuller puller def pipeConnection = Mock(PipeConnection) { - registerPusher(_) >> { args -> - pusher = args[0] + registerPuller(_) >> { args -> + puller = args[0] } } def out = new PipelineOutput(pipeConnection) @@ -79,8 +79,8 @@ class PipelineOutputTest extends Specification { new Thread({ 10.times { - def nextLoad = pusher.onNextLoad() - pusher.onAccept() + def nextLoad = puller.onNextLoad() + puller.onAccept() loadTransfer << nextLoad cdlWorkDone.countDown() From d2eaa3c47855cae788ec422ab5ea88d2347d1169 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 19 Feb 2023 19:23:09 +0100 Subject: [PATCH 20/45] [ADD] Working version of Grpc Pipeline Backend ;) [TAG] Pipelines #14 --- docs/pipeline/kafka-setup/ingest.yml | 11 +++ proto-doer/src/main/proto/pipeline.proto | 12 +++ src/main/java/io/github/s7i/doer/Doer.java | 2 + .../io/github/s7i/doer/command/Ingest.java | 14 --- .../doer/command/util/CommandManifest.java | 79 ++++++++------- .../io/github/s7i/doer/command/util/Misc.java | 12 +-- .../doer/domain/ingest/IngestProcessor.java | 28 ++++-- .../output/creator/PipelineOutputCreator.java | 4 +- .../s7i/doer/pipeline/BlockingPipePuller.java | 2 +- .../s7i/doer/pipeline/GrpcConnection.java | 99 +++++++++++++++++++ .../io/github/s7i/doer/pipeline/Pipeline.java | 15 ++- .../s7i/doer/pipeline/PipelineService.java | 20 +++- 12 files changed, 225 insertions(+), 73 deletions(-) create mode 100644 docs/pipeline/kafka-setup/ingest.yml create mode 100644 src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java diff --git a/docs/pipeline/kafka-setup/ingest.yml b/docs/pipeline/kafka-setup/ingest.yml new file mode 100644 index 0000000..56a4872 --- /dev/null +++ b/docs/pipeline/kafka-setup/ingest.yml @@ -0,0 +1,11 @@ +version: v1 +kind: ingest +param: + doer.output: pipeline://grpc + doer.pipeline.backend: grpc + doer.pipeline.backend.target: localhost:6565 +ingest: + - record: record 1 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + - record: record 2 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + - record: record 3 ${date:yyyy-MM-dd}T${date:HH:mm:ss}+00:00 + key: key-record 3 diff --git a/proto-doer/src/main/proto/pipeline.proto b/proto-doer/src/main/proto/pipeline.proto index f58ae9c..7b24cd4 100644 --- a/proto-doer/src/main/proto/pipeline.proto +++ b/proto-doer/src/main/proto/pipeline.proto @@ -7,9 +7,21 @@ import "google/protobuf/any.proto"; option java_multiple_files = true; service PipelineService { + rpc exchangeMeta(MetaOp) returns (MetaOp); rpc publish(PipelinePublishRequest) returns (PipelinePublishResponse); } +message MetaOp { + message Request { + string name = 1; + } + message Response { + string status = 1; + } + Request request = 1; + Response response = 2; +} + message PipelinePublishRequest { PipelineLoad pipelineLoad = 1; } diff --git a/src/main/java/io/github/s7i/doer/Doer.java b/src/main/java/io/github/s7i/doer/Doer.java index 1c66d91..82ca392 100644 --- a/src/main/java/io/github/s7i/doer/Doer.java +++ b/src/main/java/io/github/s7i/doer/Doer.java @@ -6,6 +6,7 @@ import io.github.s7i.doer.command.util.Misc; import io.github.s7i.doer.domain.ServiceEntrypoint; import io.github.s7i.doer.domain.grpc.GrpcServer; +import io.github.s7i.doer.pipeline.PipelineService; import io.github.s7i.doer.util.Banner; import io.github.s7i.doer.util.GitProps; import org.slf4j.Logger; @@ -24,6 +25,7 @@ Rocks.class, GrpcHealth.class, ZooSrv.class, + PipelineService.class, Misc.class}) public class Doer implements Runnable, Banner { diff --git a/src/main/java/io/github/s7i/doer/command/Ingest.java b/src/main/java/io/github/s7i/doer/command/Ingest.java index f7477b6..8c1a0fd 100644 --- a/src/main/java/io/github/s7i/doer/command/Ingest.java +++ b/src/main/java/io/github/s7i/doer/command/Ingest.java @@ -1,29 +1,15 @@ package io.github.s7i.doer.command; import io.github.s7i.doer.Context; -import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.ingest.IngestProcessor; import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; import picocli.CommandLine.Command; -import picocli.CommandLine.Option; import java.io.File; -import java.util.concurrent.Callable; @Command(name = "ingest") public class Ingest extends ManifestFileCommand { - @Option(names = {"-y"}, required = true) - File yaml; - - @Override - public File getYamlFile() { - if (!yaml.exists()) { - throw new IllegalStateException("The manifest file doesn't exists: " + yaml); - } - return yaml; - } - @Override protected File getDefaultManifestFile() { return new File("ingest.yml"); diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 181b949..612c531 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -4,22 +4,25 @@ import io.github.s7i.doer.command.Ingest; import io.github.s7i.doer.command.KafkaFeeder; import io.github.s7i.doer.command.dump.KafkaDump; +import io.github.s7i.doer.domain.ConfigProcessor; +import io.github.s7i.doer.util.Banner; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import picocli.CommandLine; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.time.Duration; import java.time.Instant; import java.util.List; +import java.util.Optional; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; - -import io.github.s7i.doer.util.Banner; -import io.github.s7i.doer.domain.ConfigProcessor; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import picocli.CommandLine; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.github.s7i.doer.command.ManifestFileCommand.Builder.fromManifestFile; import static java.util.Objects.requireNonNull; @@ -68,41 +71,45 @@ public void run() { } } + private Optional mapToTask(File yaml) { + try (var br = Files.newBufferedReader(yaml.toPath())) { + String version = br.readLine(); + String kind = br.readLine(); + kind = kind.substring(kind.lastIndexOf(':') + 1).trim(); + + switch (kind) { + case "kafka-ingest": + return Optional.of(new TaskWrapper(kind, fromManifestFile(KafkaFeeder.class, yaml))); + case "kafka-dump": + return Optional.of(new TaskWrapper(kind, fromManifestFile(KafkaDump.class, yaml))); + case "ingest": + return Optional.of(new TaskWrapper(kind, fromManifestFile(Ingest.class, yaml))); + case "config": + return Optional.of(() -> new ConfigProcessor(yaml).processConfig()); + default: + log.warn("unsupported kind/type of file: {}", kind); + break; + } + + } catch (IOException e) { + log.error("reading yaml file", e); + } + return Optional.empty(); + } + @Override public void run() { printBanner(); requireNonNull(yamls, "manifest file set..."); - var pool = Executors.newFixedThreadPool(Math.min(yamls.length, MAX_THR_COUNT), this::spawnNewThread); - for (var yaml : yamls) { - try (var br = Files.newBufferedReader(yaml.toPath())) { - String version = br.readLine(); - String kind = br.readLine(); - kind = kind.substring(kind.lastIndexOf(':') + 1).trim(); - - switch (kind) { - case "kafka-ingest": - pool.execute(new TaskWrapper(kind, fromManifestFile(KafkaFeeder.class, yaml))); - break; - case "kafka-dump": - pool.execute(new TaskWrapper(kind, fromManifestFile(KafkaDump.class, yaml))); - break; - case "ingest": - pool.execute(new TaskWrapper(kind, fromManifestFile(Ingest.class, yaml))); - break; - case "config": - pool.execute(() -> new ConfigProcessor(yaml).processConfig()); - break; - default: - log.warn("unsupported kind/type of file: {}", kind); - break; + var tasks = Stream.of(yamls) + .map(this::mapToTask) + .flatMap(Optional::stream) + .collect(Collectors.toList()); - } + var pool = Executors.newFixedThreadPool(Math.min(tasks.size(), MAX_THR_COUNT), this::spawnNewThread); + tasks.forEach(pool::execute); - } catch (IOException e) { - log.error("reading yaml file", e); - } - } try { pool.shutdown(); pool.awaitTermination(24, TimeUnit.HOURS); diff --git a/src/main/java/io/github/s7i/doer/command/util/Misc.java b/src/main/java/io/github/s7i/doer/command/util/Misc.java index 1f57163..47228f8 100644 --- a/src/main/java/io/github/s7i/doer/command/util/Misc.java +++ b/src/main/java/io/github/s7i/doer/command/util/Misc.java @@ -4,7 +4,6 @@ import com.google.protobuf.TextFormat; import io.github.s7i.doer.Doer; import io.github.s7i.doer.command.file.ReplaceInFile; -import io.github.s7i.doer.pipeline.PipelineService; import io.github.s7i.doer.util.GitProps; import io.github.s7i.doer.util.PropertyResolver; import io.github.s7i.doer.util.Utils; @@ -21,11 +20,13 @@ import org.mvel2.MVEL; import org.mvel2.compiler.CompiledExpression; import picocli.CommandLine; -import picocli.CommandLine.Parameters; -import picocli.CommandLine.Option; import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; -import java.io.*; +import java.io.File; +import java.io.IOException; +import java.io.StringReader; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; @@ -42,8 +43,7 @@ description = "Miscellaneous command set.", subcommands = { ReplaceInFile.class, - CommandManifest.class, - PipelineService.class + CommandManifest.class } ) @Slf4j(topic = "doer.console") diff --git a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java index 7e71aae..8fab5c8 100644 --- a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java @@ -1,10 +1,11 @@ package io.github.s7i.doer.domain.ingest; import io.github.s7i.doer.Context; +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.output.ConsoleOutput; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; -import io.github.s7i.doer.util.PropertyResolver; +import io.github.s7i.doer.util.Mark; import lombok.RequiredArgsConstructor; import java.nio.charset.StandardCharsets; @@ -12,17 +13,26 @@ @RequiredArgsConstructor public class IngestProcessor { + @Mark.Param + public static final String DOER_OUTPUT = "doer.output"; private final Context context; - private Output output = new ConsoleOutput(); public void process(IngestRecordManifest manifest) { + try (var output = context.getParams().entrySet().stream() + .filter(e -> e.getKey().equals(DOER_OUTPUT)) + .map(def -> context.buildOutput(def::getValue)) + .findFirst() + .orElseGet(ConsoleOutput::new)) { - var propertyResolver = new PropertyResolver(context.getParams()); - - manifest.getRecords().stream() - .map(rec -> Output.Load.builder() - .data(propertyResolver.resolve(rec.getRecord()).getBytes(StandardCharsets.UTF_8)) - .build()) - .forEach(output::emit); + manifest.getRecords().stream() + .map(rec -> Output.Load.builder() + .data(context.getPropertyResolver() + .resolve(rec.getRecord()) + .getBytes(StandardCharsets.UTF_8)) + .build()) + .forEach(output::emit); + } catch (Exception e) { + throw new DoerException(e); + } } } diff --git a/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java index 70f91a1..2ce7ce4 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java +++ b/src/main/java/io/github/s7i/doer/domain/output/creator/PipelineOutputCreator.java @@ -11,6 +11,8 @@ public interface PipelineOutputCreator extends OutputCreator { @Override default Output create() { - return new PipelineOutput(getLoadPipe()); + var pipelineOutput = new PipelineOutput(getLoadPipe()); + pipelineOutput.open(); + return pipelineOutput; } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java index c061ddc..9d88969 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java @@ -21,7 +21,7 @@ public class BlockingPipePuller implements PipePuller { public Output.Load onNextLoad() { Output.Load load; - while ((load = queue.peek()) == null) { + while ((load = queue.peek()) == null && !Thread.currentThread().isInterrupted()) { try { TimeUnit.MILLISECONDS.sleep(SLEEP_FOR_CHANGE); } catch (InterruptedException e) { diff --git a/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java new file mode 100644 index 0000000..5b8c22d --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java @@ -0,0 +1,99 @@ +package io.github.s7i.doer.pipeline; + +import com.google.protobuf.Any; +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.pipeline.proto.*; +import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; +import io.grpc.ManagedChannel; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; + +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; + +@RequiredArgsConstructor +@Slf4j +public class GrpcConnection implements PipeConnection, AutoCloseable { + + public static final int TIMEOUT = 10; + private ManagedChannel channel; + private PipelineServiceGrpc.PipelineServiceFutureStub service; + private String uuid; + private Thread thread; + + final String target; + public void connect() { + channel = Grpc.newChannelBuilder( + requireNonNull(target, "target"), + InsecureChannelCredentials.create() + ).build(); + service = PipelineServiceGrpc.newFutureStub(channel); + } + + @Override + public void registerPuller(PipePuller puller) { + try { + connect(); + var meta = meteNewClient(); + uuid = service.exchangeMeta(meta).get(TIMEOUT, TimeUnit.SECONDS).getResponse().getStatus(); + + start(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + var load = puller.onNextLoad(); + sendLoad(load).ifPresentOrElse(r -> puller.onAccept(), () -> log.error("!!! BUG !!! can't happened")); + } + } catch (Exception e) { + log.error("oops", e); + } + log.debug("pipeline puller: end of work"); + }); + + } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { + log.error("oops", e); + } + } + + @NotNull + private static MetaOp meteNewClient() { + return MetaOp.newBuilder() + .setRequest(MetaOp.Request.newBuilder().setName("add-new-pipeline-client")) + .build(); + } + + Optional sendLoad(Output.Load load) { + requireNonNull(load, "load"); + try { + var request = PipelinePublishRequest.newBuilder() + .setPipelineLoad(PipelineLoad.newBuilder() + .putHeaders("client.uuid", uuid) + .setLoad(Any.pack(load.toRecord()))) + .build(); + return Optional.ofNullable(service.publish(request).get(TIMEOUT, TimeUnit.SECONDS)); + } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { + log.error("oops", e); + } + return Optional.empty(); + } + + private void start(Runnable task) { + thread = new Thread(task, "grpc-pipeline-puller"); + thread.setDaemon(true); + thread.start(); + } + + @Override + public void close() throws Exception { + if (nonNull(thread)) { + thread.interrupt(); + } + channel.shutdownNow().awaitTermination(30, TimeUnit.SECONDS); + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index f82087c..35a3a84 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -4,6 +4,7 @@ import io.github.s7i.doer.util.Mark; import lombok.extern.slf4j.Slf4j; +import java.util.EnumMap; import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -22,13 +23,23 @@ public static void initFrom(Supplier> params) { Globals.INSTANCE.getPipeline().init(p); } + enum PipelineKind { + GRPC + } - void init(Map params) { + private Map> factory = new EnumMap<>(PipelineKind.class); + void init(Map params) { + var kind = PipelineKind.valueOf(params.get(DOER_PIPELINE).toUpperCase()); + switch (kind) { + case GRPC: + factory.put(kind, () -> new GrpcConnection(params.get(DOER_PIPELINE +".target")) ); + break; + } } public PipeConnection connect(String name) { - return null; + return factory.get(PipelineKind.GRPC).get(); } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index 2fcfd0f..76fbadd 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -5,10 +5,7 @@ import com.google.protobuf.BytesValue; import com.google.protobuf.StringValue; import io.github.s7i.doer.domain.grpc.GrpcServer; -import io.github.s7i.doer.pipeline.proto.PipelineLoad; -import io.github.s7i.doer.pipeline.proto.PipelinePublishRequest; -import io.github.s7i.doer.pipeline.proto.PipelinePublishResponse; -import io.github.s7i.doer.pipeline.proto.PipelineServiceGrpc; +import io.github.s7i.doer.pipeline.proto.*; import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; import lombok.SneakyThrows; @@ -19,6 +16,7 @@ import java.io.IOException; import java.util.Map; +import java.util.UUID; import static java.util.Objects.nonNull; @@ -46,6 +44,20 @@ public void publish(PipelinePublishRequest request, StreamObserver responseObserver) { + + var response = MetaOp.newBuilder(); + + if (request.getRequest().getName().equals("add-new-pipeline-client")) { + log.info("new client"); + response.setResponse(MetaOp.Response.newBuilder().setStatus(UUID.randomUUID().toString())); + } + + responseObserver.onNext(response.build()); + responseObserver.onCompleted(); + } } /** From 931dd3a49a1a5bf844d14c702d46117c2a0e2070 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 19 Feb 2023 22:16:31 +0100 Subject: [PATCH 21/45] [REWORK] more improvements, test fixes [TAG] Pipelines #14 --- .../{kafka-setup => grpc-backend}/ingest.yml | 0 src/main/java/io/github/s7i/doer/Context.java | 36 +++++--- src/main/java/io/github/s7i/doer/Doer.java | 3 +- .../doer/command/util/CommandManifest.java | 31 +++++-- .../s7i/doer/pipeline/BackendFactory.java | 7 ++ .../s7i/doer/pipeline/GrpcConnection.java | 87 ++++++++++++++----- .../s7i/doer/pipeline/PipeConnection.java | 2 - .../io/github/s7i/doer/pipeline/Pipeline.java | 27 ++++-- .../s7i/doer/pipeline/PipelineService.java | 2 +- src/main/resources/logback.xml | 1 + 10 files changed, 146 insertions(+), 50 deletions(-) rename docs/pipeline/{kafka-setup => grpc-backend}/ingest.yml (100%) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/BackendFactory.java diff --git a/docs/pipeline/kafka-setup/ingest.yml b/docs/pipeline/grpc-backend/ingest.yml similarity index 100% rename from docs/pipeline/kafka-setup/ingest.yml rename to docs/pipeline/grpc-backend/ingest.yml diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index c7af59f..1381f3a 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -1,20 +1,24 @@ package io.github.s7i.doer; -import static io.github.s7i.doer.Doer.console; -import static java.util.Objects.requireNonNull; - -import io.github.s7i.doer.domain.output.*; +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.domain.output.OutputBuilder; +import io.github.s7i.doer.domain.output.OutputFactory; +import io.github.s7i.doer.domain.output.OutputProvider; import io.github.s7i.doer.pipeline.Pipeline; import io.github.s7i.doer.util.ParamFlagExtractor; import io.github.s7i.doer.util.PropertyResolver; import io.github.s7i.doer.util.QuitWatcher; -import java.nio.file.Path; -import java.util.Collections; -import java.util.Map; import lombok.Builder; import lombok.Builder.Default; import lombok.Getter; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +import static io.github.s7i.doer.Doer.console; +import static java.util.Objects.requireNonNull; + public interface Context extends ParamFlagExtractor { @Builder @@ -29,7 +33,7 @@ class InitialParameters { class Initializer { static { - Runtime.getRuntime().addShutdownHook(new Thread(Initializer::shutdown, "shutdown")); + Runtime.getRuntime().addShutdownHook(new Thread(Globals.INSTANCE::stopAll, "shutdown")); } public Initializer(InitialParameters parameters) { @@ -44,18 +48,22 @@ public Initializer(InitialParameters parameters) { public Context context() { return Globals.INSTANCE; } - - private static void shutdown() { - console().info("Init shutdown procedure..."); - Globals.INSTANCE.stopHooks.forEach(Runnable::run); - console().info("Shutdown completed."); - } } default void addStopHook(Runnable runnable) { Globals.INSTANCE.stopHooks.add(runnable); } + default void stopAll() { + console().info("Init shutdown procedure..."); + Globals.INSTANCE.stopHooks.forEach(Runnable::run); + console().info("Shutdown completed."); + } + + default void stopAllSilent() { + Globals.INSTANCE.stopHooks.forEach(Runnable::run); + } + default OutputFactory getOutputFactory() { return Globals.INSTANCE.getScope().getOutputFactory(); } diff --git a/src/main/java/io/github/s7i/doer/Doer.java b/src/main/java/io/github/s7i/doer/Doer.java index 82ca392..453cf0f 100644 --- a/src/main/java/io/github/s7i/doer/Doer.java +++ b/src/main/java/io/github/s7i/doer/Doer.java @@ -29,7 +29,8 @@ Misc.class}) public class Doer implements Runnable, Banner { - static final Logger CONSOLE = LoggerFactory.getLogger("doer.console"); + public static final String DOER_CONSOLE = "doer.console"; + static final Logger CONSOLE = LoggerFactory.getLogger(DOER_CONSOLE); public static final String FLAGS = "doer.flags"; public static final String FLAG_USE_TRACING = "trace"; public static final String FLAG_SEND_AND_FORGET = "send-and-forget"; diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 612c531..61eda4f 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -1,5 +1,6 @@ package io.github.s7i.doer.command.util; +import io.github.s7i.doer.Globals; import io.github.s7i.doer.command.Command; import io.github.s7i.doer.command.Ingest; import io.github.s7i.doer.command.KafkaFeeder; @@ -24,6 +25,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static io.github.s7i.doer.Doer.DOER_CONSOLE; import static io.github.s7i.doer.command.ManifestFileCommand.Builder.fromManifestFile; import static java.util.Objects.requireNonNull; @@ -32,7 +34,7 @@ aliases = "cm", description = "Parsing command manifest yaml file." ) -@Slf4j +@Slf4j(topic = DOER_CONSOLE) public class CommandManifest implements Runnable, Banner { public static final int MAX_THR_COUNT = 20; @@ -53,6 +55,7 @@ static class TaskWrapper implements Runnable { final String name; final Command coreTask; + Duration duration; @Override public void run() { @@ -65,10 +68,12 @@ public void run() { log.error("fatal error", e); } } finally { - var duration = Duration.between(begin, Instant.now()); - log.info("Task {} ends in {}", name, duration); + duration = Duration.between(begin, Instant.now()); } } + public String getSummary() { + return String.format("Task %s ends in %s", name, duration); + } } private Optional mapToTask(File yaml) { @@ -113,9 +118,25 @@ public void run() { try { pool.shutdown(); pool.awaitTermination(24, TimeUnit.HOURS); - log.info("All task ended"); + + log.info("All task finished."); + log.info("Summary:"); + + var sum = tasks.stream() + .filter(t -> t instanceof TaskWrapper) + .mapToLong(t -> { + var task = (TaskWrapper) t; + log.info(task.getSummary()); + return task.duration.toNanos(); + }) + .sum(); + + log.info("Total time: {}.", Duration.ofNanos(sum)); + + Globals.INSTANCE.stopAllSilent(); + } catch (InterruptedException e) { - log.error("", e); + log.error("oops", e); Thread.currentThread().interrupt(); } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/BackendFactory.java b/src/main/java/io/github/s7i/doer/pipeline/BackendFactory.java new file mode 100644 index 0000000..bc898b5 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/BackendFactory.java @@ -0,0 +1,7 @@ +package io.github.s7i.doer.pipeline; + +public interface BackendFactory { + + PipeConnection create(); + +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java index 5b8c22d..02a1ae9 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java @@ -1,14 +1,15 @@ package io.github.s7i.doer.pipeline; import com.google.protobuf.Any; +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.pipeline.proto.*; import io.grpc.Grpc; import io.grpc.InsecureChannelCredentials; import io.grpc.ManagedChannel; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.jetbrains.annotations.NotNull; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -28,6 +29,8 @@ public class GrpcConnection implements PipeConnection, AutoCloseable { private String uuid; private Thread thread; + private transient boolean isClosed; + final String target; public void connect() { channel = Grpc.newChannelBuilder( @@ -35,34 +38,64 @@ public void connect() { InsecureChannelCredentials.create() ).build(); service = PipelineServiceGrpc.newFutureStub(channel); + + int retryNo=0; + final int ofRetries=10; + + boolean success = false; + do { + try { + success = introduceNewConnection(); + + } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { + log.warn("oops but still working... {}/{}", retryNo + 1, ofRetries, e); + try { + int t = 1000 + (1000 * retryNo); + TimeUnit.MILLISECONDS.sleep(t); + } catch (InterruptedException x) { + Thread.currentThread().interrupt(); + break; + } + } + } while (!success && retryNo++ < ofRetries ); + + if (!success) { + throw new DoerException("[PIPELINE GRPC] FAILURE"); + } + } + + boolean introduceNewConnection() throws ExecutionException, InterruptedException, TimeoutException { + uuid = service.exchangeMeta(metaNewConnection()) + .get(TIMEOUT, TimeUnit.SECONDS) + .getResponse() + .getStatus(); + log.info("pipeline client id: {}", uuid); + return true; } @Override public void registerPuller(PipePuller puller) { - try { - connect(); - var meta = meteNewClient(); - uuid = service.exchangeMeta(meta).get(TIMEOUT, TimeUnit.SECONDS).getResponse().getStatus(); + connect(); - start(() -> { - try { - while (!Thread.currentThread().isInterrupted()) { - var load = puller.onNextLoad(); - sendLoad(load).ifPresentOrElse(r -> puller.onAccept(), () -> log.error("!!! BUG !!! can't happened")); + start(() -> { + try { + while (!Thread.currentThread().isInterrupted() && !isClosed) { + var load = puller.onNextLoad(); + if (nonNull(load)) { + sendLoad(load).ifPresentOrElse( + r -> puller.onAccept(), + () -> log.error("!!! BUG !!! can't happened") + ); } - } catch (Exception e) { - log.error("oops", e); } - log.debug("pipeline puller: end of work"); - }); - - } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { - log.error("oops", e); - } + } catch (Exception e) { + log.error("oops", e); + } + log.debug("[PIPELINE GRPC] : pipeline puller - end of work"); + }); } - @NotNull - private static MetaOp meteNewClient() { + private static MetaOp metaNewConnection() { return MetaOp.newBuilder() .setRequest(MetaOp.Request.newBuilder().setName("add-new-pipeline-client")) .build(); @@ -85,12 +118,24 @@ Optional sendLoad(Output.Load load) { private void start(Runnable task) { thread = new Thread(task, "grpc-pipeline-puller"); - thread.setDaemon(true); + //thread.setDaemon(true); thread.start(); } + + @SneakyThrows + public void closeSafe() { + close(); + } + @Override public void close() throws Exception { + log.debug("[PIPELINE GRPC] : closing"); + + TimeUnit.MILLISECONDS.sleep(1000); + + isClosed = true; + if (nonNull(thread)) { thread.interrupt(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index 5148445..56f1fef 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -3,6 +3,4 @@ public interface PipeConnection { void registerPuller(PipePuller puller); - - //void registerPuller(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 35a3a84..13ebbce 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -9,6 +9,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static java.util.Objects.requireNonNull; + @Slf4j public class Pipeline { @@ -16,30 +18,43 @@ public class Pipeline { public static final String DOER_PIPELINE = "doer.pipeline.backend"; public static void initFrom(Supplier> params) { - var p = params.get().entrySet() + var setup = params.get().entrySet() .stream() .filter(es -> es.getKey().startsWith(DOER_PIPELINE)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - Globals.INSTANCE.getPipeline().init(p); + + if (!setup.isEmpty()) { + Globals.INSTANCE.getPipeline().init(setup); + } } enum PipelineKind { GRPC } - private Map> factory = new EnumMap<>(PipelineKind.class); + private Map register = new EnumMap<>(PipelineKind.class); void init(Map params) { - var kind = PipelineKind.valueOf(params.get(DOER_PIPELINE).toUpperCase()); + var backend = requireNonNull(params.get(DOER_PIPELINE), DOER_PIPELINE).toUpperCase(); + + var kind = PipelineKind.valueOf(backend); switch (kind) { case GRPC: - factory.put(kind, () -> new GrpcConnection(params.get(DOER_PIPELINE +".target")) ); + register.put(kind, () -> { + var c = new GrpcConnection(params.get(DOER_PIPELINE +".target")); + Globals.INSTANCE.addStopHook(c::closeSafe); + return c; + }); break; } } public PipeConnection connect(String name) { - return factory.get(PipelineKind.GRPC).get(); + return register.get(PipelineKind.GRPC).create(); + } + + public boolean isEnabled() { + return !register.isEmpty(); } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index 76fbadd..f233523 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -20,7 +20,7 @@ import static java.util.Objects.nonNull; -@Command(name = "pipeline") +@Command(name = "pipeline", description = "Pipeline Backend Service.") @Slf4j(topic = "doer.console") public class PipelineService implements Runnable { diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml index 69637f7..c30eeea 100644 --- a/src/main/resources/logback.xml +++ b/src/main/resources/logback.xml @@ -23,6 +23,7 @@ + From 1a8723688f7e79cd379495c591640bc1c123b287 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 00:16:35 +0100 Subject: [PATCH 22/45] [ADD] Pipe storage, heap based [TAG] Pipelines #14 --- .../doer/pipeline/store/PipelineStorage.java | 165 ++++++++++++++++++ .../pipeline/store/PipelineStorageTest.groovy | 66 +++++++ 2 files changed, 231 insertions(+) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java create mode 100644 src/test/groovy/io/github/s7i/doer/pipeline/store/PipelineStorageTest.groovy diff --git a/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java new file mode 100644 index 0000000..90dbe50 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java @@ -0,0 +1,165 @@ +package io.github.s7i.doer.pipeline.store; + +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import lombok.experimental.Accessors; + +import java.util.*; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static java.util.Objects.*; + +public class PipelineStorage

{ + + public enum Direction { + FIFO, LIFO; + + Optional> from(Element e) { + requireNonNull(e, "element"); + switch (this) { + case FIFO: + return e.getPrev(); + case LIFO: + return e.getNext(); + } + throw new IllegalStateException(); + } + } + + public interface OffsetSequence { + Long nextOffset(); + } + + static class PipeIterator

implements Iterator> { + + Element

current; + Element

initial; + final Direction direction; + + PipeIterator(Direction direction, Element

initial) { + requireNonNull(initial, "initial"); + requireNonNull(direction, "direction"); + + this.direction = direction; + this.initial = initial; + } + + @Override + public boolean hasNext() { + if (nonNull(initial)) { + return true; + } + + return nonNull(current) && direction.from(current).isPresent(); + } + + @Override + public Element

next() { + if (nonNull(initial)) { + + current = initial; + initial = null; + + return current; + } + + var nextStep = direction.from(requireNonNull(current, + "missing iterator element" + )); + current = nextStep.get(); + return current; + } + } + + @Data + @Accessors(fluent = true) + public static class Pipe

{ + Element

tail; + Element

head; + + public void putElementWithPackage(P pack, OffsetSequence oseq) { + var link = new Link

(); + var el = new Element

(link, oseq.nextOffset(), pack); + + if (nonNull(tail)) { + tail.link.prev = el; + link.next = tail; + } + tail = el; + + if (isNull(head)) { + head = el; + } + } + public Iterable> iterator(Direction direction) { + Supplier>> prov = () -> new PipeIterator<>( + direction, + direction == Direction.FIFO ? head : tail + ); + + return prov::get; + } + + public Stream> stream(Direction direction) { + return StreamSupport.stream(iterator(direction).spliterator(), false); + } + } + + @RequiredArgsConstructor + @ToString + public static class Element

{ + + final Link

link; + + final Long offset; + final P pack; + + public P getPackage() { + return pack; + } + + public Optional> getPrev() { + return Optional.ofNullable(link.prev); + } + + public Optional> getNext() { + return Optional.ofNullable(link.next); + } + } + + + static class Link

{ + Element

prev; + Element

next; + + @Override + public String toString() { + return String.format("Link[ prev: %d, next: %d ]", + nonNull(prev) ? prev.offset : -1, + nonNull(next) ? next.offset : -1 + ); + } + } + + private Map> store = new HashMap<>(); + private Long offsetSequence = 0L; + + private Long nextOffset() { + return offsetSequence++; + } + + public Pipe

addToPipe(String pipeName, Collection

packages) { + var pipe = store.computeIfAbsent(pipeName, name -> new Pipe<>()); + for (var pack : packages) { + pipe.putElementWithPackage(pack, this::nextOffset); + } + return pipe; + } + + public Pipe

addToPipe(String pipeName, P pack) { + return addToPipe(pipeName, List.of(pack)); + } +} diff --git a/src/test/groovy/io/github/s7i/doer/pipeline/store/PipelineStorageTest.groovy b/src/test/groovy/io/github/s7i/doer/pipeline/store/PipelineStorageTest.groovy new file mode 100644 index 0000000..fa79f9d --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/pipeline/store/PipelineStorageTest.groovy @@ -0,0 +1,66 @@ +package io.github.s7i.doer.pipeline.store + +import spock.lang.Specification + +import java.util.stream.Collectors + +class PipelineStorageTest extends Specification { + def "AddToPipe - one element"() { + given: + def ps = new PipelineStorage() + when: + def pipe = ps.addToPipe("pipe-name", "pack-of-data") + + then: + pipe + + pipe.head.offset == 0 + pipe.tail.getPackage() == "pack-of-data" + } + + + def "AddToPipe - 5 elements"() { + given: + def ps = new PipelineStorage() + def els = [] + 5.times { + els << "package-of-data-${it + 1}" + } + + when: + def pipe = ps.addToPipe("pipe-name", els) + + then: + pipe + + pipe.tail.getPackage() == "package-of-data-5" + pipe.head.getPackage() == "package-of-data-1" + + pipe.tail.offset == 4L + pipe.head.offset == 0L + } + + def "Pipe - iterate"() { + given: + def ps = new PipelineStorage() + def els = [] + 5.times { + els << "data-${it + 1}" + } + + when: + def pipe = ps.addToPipe("pipe-name", els) + def collected = pipe.stream(direction) + .map { it.getPackage() } + .collect(Collectors.toList()) + + then: + collected == expected + + where: + direction | expected + PipelineStorage.Direction.FIFO | ["data-1", "data-2", "data-3", "data-4", "data-5"] + PipelineStorage.Direction.LIFO | ["data-5", "data-4", "data-3", "data-2", "data-1"] + } + +} From 78927abbdf32fd37643254da7899499a6d670f85 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 13:39:00 +0100 Subject: [PATCH 23/45] [REWORK] exit with code from command execution result --- src/main/java/io/github/s7i/doer/Doer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/github/s7i/doer/Doer.java b/src/main/java/io/github/s7i/doer/Doer.java index 453cf0f..59a2226 100644 --- a/src/main/java/io/github/s7i/doer/Doer.java +++ b/src/main/java/io/github/s7i/doer/Doer.java @@ -78,8 +78,10 @@ public static void main(String[] args) { ? new CommandManifest() : new Doer(); - new CommandLine(command) + var exitCode = new CommandLine(command) .setCaseInsensitiveEnumValuesAllowed(true) .execute(args); + + System.exit(exitCode); } } From 72fc016f454f6d934bd68e4856dac515c40d146b Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 13:43:28 +0100 Subject: [PATCH 24/45] [REWORK] parameter map fields for more future general usage [TAG] Pipelines #14 --- proto-doer/src/main/proto/pipeline.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto-doer/src/main/proto/pipeline.proto b/proto-doer/src/main/proto/pipeline.proto index 7b24cd4..a26a843 100644 --- a/proto-doer/src/main/proto/pipeline.proto +++ b/proto-doer/src/main/proto/pipeline.proto @@ -14,9 +14,11 @@ service PipelineService { message MetaOp { message Request { string name = 1; + map parameters = 2; } message Response { string status = 1; + map parameters = 2; } Request request = 1; Response response = 2; From 95431a9bd44e6cafbfea699c6b9b302f49ef9c9d Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 13:43:59 +0100 Subject: [PATCH 25/45] [REWORK] IDEA run configuration [TAG] Pipelines #14 --- ...un.xml => Doer pipeline grpc service.run.xml} | 6 +++--- .run/Doer pipeline ingest.run.xml | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) rename .run/{Doer kfeed - simple ingest.run.xml => Doer pipeline grpc service.run.xml} (58%) create mode 100644 .run/Doer pipeline ingest.run.xml diff --git a/.run/Doer kfeed - simple ingest.run.xml b/.run/Doer pipeline grpc service.run.xml similarity index 58% rename from .run/Doer kfeed - simple ingest.run.xml rename to .run/Doer pipeline grpc service.run.xml index 6c234f8..b4761c4 100644 --- a/.run/Doer kfeed - simple ingest.run.xml +++ b/.run/Doer pipeline grpc service.run.xml @@ -1,11 +1,11 @@ - + 

+ *     "Dependency injection is a kind of globals with configuration."
+ *
+ */ @Slf4j public enum Globals implements Context { INSTANCE; diff --git a/src/main/java/io/github/s7i/doer/command/CommandWithContext.java b/src/main/java/io/github/s7i/doer/command/CommandWithContext.java new file mode 100644 index 0000000..aa7697a --- /dev/null +++ b/src/main/java/io/github/s7i/doer/command/CommandWithContext.java @@ -0,0 +1,23 @@ +package io.github.s7i.doer.command; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.config.Base; + +public abstract class CommandWithContext extends ManifestFileCommand { + + protected abstract Class manifestClass(); + + @Override + public void onExecuteCommand() { + var manifest = parseYaml(manifestClass()); + + var ctx = new Context.Initializer(Context.InitialParameters.builder() + .workDir(yaml.toPath().toAbsolutePath().getParent()) + .params(manifest.getParams()) + .build()) + .context(); + onExecuteCommand(ctx, manifest); + } + + public abstract void onExecuteCommand(Context context, M manifest); +} diff --git a/src/main/java/io/github/s7i/doer/command/SinkCommand.java b/src/main/java/io/github/s7i/doer/command/SinkCommand.java new file mode 100644 index 0000000..e445ac1 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/command/SinkCommand.java @@ -0,0 +1,27 @@ +package io.github.s7i.doer.command; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.domain.SinkProcessor; +import io.github.s7i.doer.manifest.SinkManifest; +import picocli.CommandLine; + +import java.io.File; + +@CommandLine.Command(name = "sink") +public class SinkCommand extends CommandWithContext { + + @Override + protected File getDefaultManifestFile() { + return new File("sink.yml"); + } + + @Override + protected Class manifestClass() { + return SinkManifest.class; + } + + @Override + public void onExecuteCommand(Context context, SinkManifest manifest) { + new SinkProcessor(context).execute(manifest.getSpec().stream().findFirst().orElseThrow()); + } +} diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java new file mode 100644 index 0000000..b5df4cf --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -0,0 +1,36 @@ +package io.github.s7i.doer.domain; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.manifest.SinkManifest; +import io.github.s7i.doer.pipeline.PipePuller; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class SinkProcessor { + + static class SinkPipePuller implements PipePuller { + + @Override + public Output.Load onNextLoad() { + return null; + } + + @Override + public void onAccept() { + + } + } + + final Context context; + private SinkPipePuller puller; + + public void execute(SinkManifest.SinkSpec sinkSpec) { + puller = new SinkPipePuller(); + + context.lookupPipeline().ifPresent(pipeline -> { + pipeline.connect("sink").registerPuller(puller); + }); + } + +} diff --git a/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java b/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java new file mode 100644 index 0000000..3a70962 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java @@ -0,0 +1,19 @@ +package io.github.s7i.doer.manifest; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.github.s7i.doer.config.Base; +import lombok.Data; +import lombok.Getter; + +import java.util.List; + +@Getter +public class SinkManifest extends Base { + @Data + public static class SinkSpec { + String output; + } + + @JsonProperty("spec") + List spec; +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index f233523..3247fb6 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -6,6 +6,7 @@ import com.google.protobuf.StringValue; import io.github.s7i.doer.domain.grpc.GrpcServer; import io.github.s7i.doer.pipeline.proto.*; +import io.github.s7i.doer.pipeline.store.PipelineStorage; import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; import lombok.SneakyThrows; @@ -33,11 +34,16 @@ public static void main(String[] args) { static class Handler extends PipelineServiceGrpc.PipelineServiceImplBase { + + PipelineStorage storage = new PipelineStorage<>(); + @Override public void publish(PipelinePublishRequest request, StreamObserver responseObserver) { var pipelineLoad = request.getPipelineLoad(); log.info("getting load: {}", pipelineLoad); + storage.addToPipe("default", pipelineLoad); + responseObserver.onNext(PipelinePublishResponse.newBuilder() .setStatus("ok") diff --git a/src/test/groovy/io/github/s7i/doer/command/SinkCommandTest.groovy b/src/test/groovy/io/github/s7i/doer/command/SinkCommandTest.groovy new file mode 100644 index 0000000..718522e --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/command/SinkCommandTest.groovy @@ -0,0 +1,16 @@ +package io.github.s7i.doer.command + +import spock.lang.Specification + +class SinkCommandTest extends Specification { + + + def "parse manifest test"() { + given: + def manifestPath = "src/test/resources/pipeline/sink.yml" + def sink = ManifestFileCommand.Builder.fromManifestFile(SinkCommand.class, new File(manifestPath)) + + expect: + sink.onExecuteCommand() + } +} From 4483c44d838ff2b7f7e3a20897d26d0f4992caab Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 13:54:02 +0100 Subject: [PATCH 27/45] [REWORK] inheritance from CommandWithManifest [TAG] Pipelines #14 --- .../java/io/github/s7i/doer/command/Ingest.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/Ingest.java b/src/main/java/io/github/s7i/doer/command/Ingest.java index 8c1a0fd..6276a2b 100644 --- a/src/main/java/io/github/s7i/doer/command/Ingest.java +++ b/src/main/java/io/github/s7i/doer/command/Ingest.java @@ -8,25 +8,20 @@ import java.io.File; @Command(name = "ingest") -public class Ingest extends ManifestFileCommand { +public class Ingest extends CommandWithContext { @Override protected File getDefaultManifestFile() { return new File("ingest.yml"); } - @Override - public void onExecuteCommand() { - - var manifest = parseYaml(IngestRecordManifest.class); - - var ctx = new Context.Initializer(Context.InitialParameters.builder() - .workDir(yaml.toPath().toAbsolutePath().getParent()) - .params(manifest.getParams()) - .build()) - .context(); + protected Class manifestClass() { + return IngestRecordManifest.class; + } + @Override + public void onExecuteCommand(Context ctx, IngestRecordManifest manifest) { new IngestProcessor(ctx).process(manifest); } } From 23e2ef9ca1051f6c9c5994218e30dfc279fbdd14 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 14:36:31 +0100 Subject: [PATCH 28/45] [REWORK] WIP / checkpoint 1 [TAG] Pipelines #14 --- .../s7i/doer/pipeline/BlockingPipePuller.java | 3 +- .../s7i/doer/pipeline/PipeConnection.java | 10 ++- .../github/s7i/doer/pipeline/PipePusher.java | 8 ++ .../io/github/s7i/doer/pipeline/Pipeline.java | 16 +++- .../doer/pipeline/grcp/GrpcConnection.java | 84 +++++++++++++++++++ .../pipeline/grcp/GrpcInboundConnection.java | 19 +++++ .../GrpcOutboundConnection.java} | 79 +++-------------- 7 files changed, 144 insertions(+), 75 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/PipePusher.java create mode 100644 src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java create mode 100644 src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java rename src/main/java/io/github/s7i/doer/pipeline/{GrpcConnection.java => grcp/GrpcOutboundConnection.java} (50%) diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java index 9d88969..0334ad3 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java @@ -7,7 +7,7 @@ import java.util.concurrent.TimeUnit; @Slf4j -public class BlockingPipePuller implements PipePuller { +public class BlockingPipePuller implements PipePuller, PipePusher { public static final int SLEEP_FOR_CHANGE = 10; private final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); @@ -44,6 +44,7 @@ public void onAccept() { * * @param load */ + @Override public void offer(Output.Load load) { boolean accepted; do { diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index 56f1fef..db0dfdf 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -1,6 +1,14 @@ package io.github.s7i.doer.pipeline; +import io.github.s7i.doer.DoerException; + public interface PipeConnection { - void registerPuller(PipePuller puller); + default void registerPuller(PipePuller puller) { + throw new DoerException("not implemented"); + } + + default void registerPusher(PipePusher pusher) { + throw new DoerException("not implemented"); + } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java new file mode 100644 index 0000000..a6a238e --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/PipePusher.java @@ -0,0 +1,8 @@ +package io.github.s7i.doer.pipeline; + +import io.github.s7i.doer.domain.output.Output; + +public interface PipePusher { + + void offer(Output.Load load); +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 13ebbce..1b2531d 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -1,6 +1,8 @@ package io.github.s7i.doer.pipeline; import io.github.s7i.doer.Globals; +import io.github.s7i.doer.pipeline.grcp.GrpcInboundConnection; +import io.github.s7i.doer.pipeline.grcp.GrpcOutboundConnection; import io.github.s7i.doer.util.Mark; import lombok.extern.slf4j.Slf4j; @@ -29,7 +31,7 @@ public static void initFrom(Supplier> params) { } enum PipelineKind { - GRPC + GRPC_OUTBOUND, GRPC_INBOUND } private Map register = new EnumMap<>(PipelineKind.class); @@ -39,18 +41,24 @@ void init(Map params) { var kind = PipelineKind.valueOf(backend); switch (kind) { - case GRPC: + case GRPC_OUTBOUND: register.put(kind, () -> { - var c = new GrpcConnection(params.get(DOER_PIPELINE +".target")); + var c = new GrpcOutboundConnection(params.get(DOER_PIPELINE +".target")); Globals.INSTANCE.addStopHook(c::closeSafe); return c; }); break; + case GRPC_INBOUND: + register.put(kind, ()-> { + var c = new GrpcInboundConnection(params.get(DOER_PIPELINE +".target")); + Globals.INSTANCE.addStopHook(c::closeSafe); + return c; + }); } } public PipeConnection connect(String name) { - return register.get(PipelineKind.GRPC).create(); + return register.get(PipelineKind.GRPC_OUTBOUND).create(); } public boolean isEnabled() { diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java new file mode 100644 index 0000000..57efd15 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java @@ -0,0 +1,84 @@ +package io.github.s7i.doer.pipeline.grcp; + +import io.github.s7i.doer.DoerException; +import io.github.s7i.doer.pipeline.PipeConnection; +import io.github.s7i.doer.pipeline.proto.MetaOp; +import io.github.s7i.doer.pipeline.proto.PipelineServiceGrpc; +import io.grpc.Grpc; +import io.grpc.InsecureChannelCredentials; +import io.grpc.ManagedChannel; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static java.util.Objects.requireNonNull; + +@RequiredArgsConstructor +@Slf4j +public abstract class GrpcConnection implements PipeConnection, AutoCloseable { + + public static final int TIMEOUT = 10; + protected ManagedChannel channel; + protected PipelineServiceGrpc.PipelineServiceFutureStub service; + + protected String uuid; + + final String target; + + public void connect() { + channel = Grpc.newChannelBuilder( + requireNonNull(target, "target"), + InsecureChannelCredentials.create() + ).build(); + service = PipelineServiceGrpc.newFutureStub(channel); + + int retryNo=0; + final int ofRetries=10; + + boolean success = false; + do { + try { + success = introduceNewConnection(); + + } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { + log.warn("oops but still working... {}/{}", retryNo + 1, ofRetries, e); + try { + int t = 1000 + (1000 * retryNo); + TimeUnit.MILLISECONDS.sleep(t); + } catch (InterruptedException x) { + Thread.currentThread().interrupt(); + break; + } + } + } while (!success && retryNo++ < ofRetries ); + + if (!success) { + throw new DoerException("[PIPELINE GRPC] FAILURE"); + } + } + + boolean introduceNewConnection() throws ExecutionException, InterruptedException, TimeoutException { + uuid = service.exchangeMeta(metaNewConnection()) + .get(TIMEOUT, TimeUnit.SECONDS) + .getResponse() + .getStatus(); + log.info("pipeline client id: {}", uuid); + return true; + } + + + static MetaOp metaNewConnection() { + return MetaOp.newBuilder() + .setRequest(MetaOp.Request.newBuilder().setName("add-new-pipeline-client")) + .build(); + } + + @SneakyThrows + public void closeSafe() { + close(); + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java new file mode 100644 index 0000000..4f3bbfc --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java @@ -0,0 +1,19 @@ +package io.github.s7i.doer.pipeline.grcp; + +import io.github.s7i.doer.pipeline.BlockingPipePuller; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class GrpcInboundConnection extends GrpcConnection { + + final BlockingPipePuller pipe = new BlockingPipePuller(); + + public GrpcInboundConnection(String target) { + super(target); + } + + @Override + public void close() throws Exception { + + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java similarity index 50% rename from src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java rename to src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java index 02a1ae9..d4af2ed 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/GrpcConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java @@ -1,14 +1,11 @@ -package io.github.s7i.doer.pipeline; +package io.github.s7i.doer.pipeline.grcp; import com.google.protobuf.Any; -import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.output.Output; -import io.github.s7i.doer.pipeline.proto.*; -import io.grpc.Grpc; -import io.grpc.InsecureChannelCredentials; -import io.grpc.ManagedChannel; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; +import io.github.s7i.doer.pipeline.PipePuller; +import io.github.s7i.doer.pipeline.proto.PipelineLoad; +import io.github.s7i.doer.pipeline.proto.PipelinePublishRequest; +import io.github.s7i.doer.pipeline.proto.PipelinePublishResponse; import lombok.extern.slf4j.Slf4j; import java.util.Optional; @@ -19,59 +16,15 @@ import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; -@RequiredArgsConstructor @Slf4j -public class GrpcConnection implements PipeConnection, AutoCloseable { +public class GrpcOutboundConnection extends GrpcConnection { - public static final int TIMEOUT = 10; - private ManagedChannel channel; - private PipelineServiceGrpc.PipelineServiceFutureStub service; - private String uuid; - private Thread thread; - - private transient boolean isClosed; - - final String target; - public void connect() { - channel = Grpc.newChannelBuilder( - requireNonNull(target, "target"), - InsecureChannelCredentials.create() - ).build(); - service = PipelineServiceGrpc.newFutureStub(channel); - - int retryNo=0; - final int ofRetries=10; - - boolean success = false; - do { - try { - success = introduceNewConnection(); - - } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { - log.warn("oops but still working... {}/{}", retryNo + 1, ofRetries, e); - try { - int t = 1000 + (1000 * retryNo); - TimeUnit.MILLISECONDS.sleep(t); - } catch (InterruptedException x) { - Thread.currentThread().interrupt(); - break; - } - } - } while (!success && retryNo++ < ofRetries ); - - if (!success) { - throw new DoerException("[PIPELINE GRPC] FAILURE"); - } + public GrpcOutboundConnection(String target) { + super(target); } - boolean introduceNewConnection() throws ExecutionException, InterruptedException, TimeoutException { - uuid = service.exchangeMeta(metaNewConnection()) - .get(TIMEOUT, TimeUnit.SECONDS) - .getResponse() - .getStatus(); - log.info("pipeline client id: {}", uuid); - return true; - } + private Thread thread; + private transient boolean isClosed; @Override public void registerPuller(PipePuller puller) { @@ -95,12 +48,6 @@ public void registerPuller(PipePuller puller) { }); } - private static MetaOp metaNewConnection() { - return MetaOp.newBuilder() - .setRequest(MetaOp.Request.newBuilder().setName("add-new-pipeline-client")) - .build(); - } - Optional sendLoad(Output.Load load) { requireNonNull(load, "load"); try { @@ -122,12 +69,6 @@ private void start(Runnable task) { thread.start(); } - - @SneakyThrows - public void closeSafe() { - close(); - } - @Override public void close() throws Exception { log.debug("[PIPELINE GRPC] : closing"); From 1637475b27b19a5e44090ef460a3a0f3fd948195 Mon Sep 17 00:00:00 2001 From: Mariusz Sygnowski Date: Sun, 19 Feb 2023 22:27:25 +0100 Subject: [PATCH 29/45] [REWORK] build updata --- .github/workflows/gradle.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index e515180..84b3afd 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -7,7 +7,7 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] + branches: [ main, release/develop ] jobs: build: @@ -23,4 +23,4 @@ jobs: - name: Grant execute permission for gradlew run: chmod +x gradlew - name: Build with Gradle - run: ./gradlew --console=plain build + run: ./gradlew -x shadowJar --console=plain build From 70c9d93db10a82bd08fef827e1ca17f5bcf1376a Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 15:55:58 +0100 Subject: [PATCH 30/45] [REWORK] WIP / checkpoint 2 [TAG] Pipelines #14 --- proto-doer/src/main/proto/pipeline.proto | 1 + .../github/s7i/doer/domain/SinkProcessor.java | 24 ++++--------- .../doer/domain/ingest/IngestProcessor.java | 13 ++----- .../domain/output/DefaultOutputProvider.java | 17 +++++++++ .../doer/domain/output/PipelineOutput.java | 7 ++-- ...ckingPipePuller.java => BlockingPipe.java} | 2 +- .../s7i/doer/pipeline/PipeConnection.java | 2 +- .../io/github/s7i/doer/pipeline/Pipeline.java | 1 + .../doer/pipeline/grcp/GrpcConnection.java | 13 +++++-- .../pipeline/grcp/GrpcInboundConnection.java | 35 +++++++++++++++++-- .../pipeline/grcp/GrpcOutboundConnection.java | 2 +- .../domain/output/PipelineOutputTest.groovy | 6 ++-- 12 files changed, 80 insertions(+), 43 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/domain/output/DefaultOutputProvider.java rename src/main/java/io/github/s7i/doer/pipeline/{BlockingPipePuller.java => BlockingPipe.java} (95%) diff --git a/proto-doer/src/main/proto/pipeline.proto b/proto-doer/src/main/proto/pipeline.proto index a26a843..89ef9ef 100644 --- a/proto-doer/src/main/proto/pipeline.proto +++ b/proto-doer/src/main/proto/pipeline.proto @@ -9,6 +9,7 @@ option java_multiple_files = true; service PipelineService { rpc exchangeMeta(MetaOp) returns (MetaOp); rpc publish(PipelinePublishRequest) returns (PipelinePublishResponse); + rpc subscribe(MetaOp) returns (stream PipelineLoad); } message MetaOp { diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java index b5df4cf..07d6d1c 100644 --- a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -1,36 +1,24 @@ package io.github.s7i.doer.domain; import io.github.s7i.doer.Context; -import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.domain.output.DefaultOutputProvider; import io.github.s7i.doer.manifest.SinkManifest; -import io.github.s7i.doer.pipeline.PipePuller; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor -public class SinkProcessor { +public class SinkProcessor implements DefaultOutputProvider { - static class SinkPipePuller implements PipePuller { - - @Override - public Output.Load onNextLoad() { - return null; - } - - @Override - public void onAccept() { - - } - } final Context context; - private SinkPipePuller puller; public void execute(SinkManifest.SinkSpec sinkSpec) { - puller = new SinkPipePuller(); context.lookupPipeline().ifPresent(pipeline -> { - pipeline.connect("sink").registerPuller(puller); + var puller = pipeline.connect("sink").lookupPuller(); + getDefaultOutput(context).emit(puller.onNextLoad()); + puller.onAccept(); }); + } } diff --git a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java index 8fab5c8..59f542d 100644 --- a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java @@ -2,27 +2,20 @@ import io.github.s7i.doer.Context; import io.github.s7i.doer.DoerException; -import io.github.s7i.doer.domain.output.ConsoleOutput; +import io.github.s7i.doer.domain.output.DefaultOutputProvider; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; -import io.github.s7i.doer.util.Mark; import lombok.RequiredArgsConstructor; import java.nio.charset.StandardCharsets; @RequiredArgsConstructor -public class IngestProcessor { +public class IngestProcessor implements DefaultOutputProvider { - @Mark.Param - public static final String DOER_OUTPUT = "doer.output"; private final Context context; public void process(IngestRecordManifest manifest) { - try (var output = context.getParams().entrySet().stream() - .filter(e -> e.getKey().equals(DOER_OUTPUT)) - .map(def -> context.buildOutput(def::getValue)) - .findFirst() - .orElseGet(ConsoleOutput::new)) { + try (var output = getDefaultOutput(context)) { manifest.getRecords().stream() .map(rec -> Output.Load.builder() diff --git a/src/main/java/io/github/s7i/doer/domain/output/DefaultOutputProvider.java b/src/main/java/io/github/s7i/doer/domain/output/DefaultOutputProvider.java new file mode 100644 index 0000000..2b44829 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/output/DefaultOutputProvider.java @@ -0,0 +1,17 @@ +package io.github.s7i.doer.domain.output; + +import io.github.s7i.doer.Context; +import io.github.s7i.doer.util.Mark; + +public interface DefaultOutputProvider { + @Mark.Param + String DOER_OUTPUT = "doer.output"; + + default Output getDefaultOutput(Context context) { + return context.getParams().entrySet().stream() + .filter(e -> e.getKey().equals(DOER_OUTPUT)) + .map(def -> context.buildOutput(def::getValue)) + .findFirst() + .orElseGet(ConsoleOutput::new); + } +} diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java index edf0eb6..5c49ff2 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -1,22 +1,21 @@ package io.github.s7i.doer.domain.output; -import io.github.s7i.doer.pipeline.BlockingPipePuller; +import io.github.s7i.doer.pipeline.BlockingPipe; import io.github.s7i.doer.pipeline.PipeConnection; import lombok.RequiredArgsConstructor; import static java.util.Objects.requireNonNull; - @RequiredArgsConstructor public class PipelineOutput implements Output { private final PipeConnection pipeConnection; - private BlockingPipePuller pusher; + private BlockingPipe pusher; @Override public void open() { - pusher = new BlockingPipePuller(); + pusher = new BlockingPipe(); pipeConnection.registerPuller(pusher); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipe.java similarity index 95% rename from src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java rename to src/main/java/io/github/s7i/doer/pipeline/BlockingPipe.java index 0334ad3..2c72334 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/BlockingPipePuller.java +++ b/src/main/java/io/github/s7i/doer/pipeline/BlockingPipe.java @@ -7,7 +7,7 @@ import java.util.concurrent.TimeUnit; @Slf4j -public class BlockingPipePuller implements PipePuller, PipePusher { +public class BlockingPipe implements PipePuller, PipePusher { public static final int SLEEP_FOR_CHANGE = 10; private final ArrayBlockingQueue queue = new ArrayBlockingQueue<>(1); diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java index db0dfdf..254fb7d 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipeConnection.java @@ -8,7 +8,7 @@ default void registerPuller(PipePuller puller) { throw new DoerException("not implemented"); } - default void registerPusher(PipePusher pusher) { + default PipePuller lookupPuller() { throw new DoerException("not implemented"); } } diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 1b2531d..00293c1 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -51,6 +51,7 @@ void init(Map params) { case GRPC_INBOUND: register.put(kind, ()-> { var c = new GrpcInboundConnection(params.get(DOER_PIPELINE +".target")); + c.connect(); Globals.INSTANCE.addStopHook(c::closeSafe); return c; }); diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java index 57efd15..bdcb097 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java @@ -23,7 +23,8 @@ public abstract class GrpcConnection implements PipeConnection, AutoCloseable { public static final int TIMEOUT = 10; protected ManagedChannel channel; - protected PipelineServiceGrpc.PipelineServiceFutureStub service; + protected PipelineServiceGrpc.PipelineServiceFutureStub serviceFuture; + protected PipelineServiceGrpc.PipelineServiceStub serviceStub; protected String uuid; @@ -34,7 +35,9 @@ public void connect() { requireNonNull(target, "target"), InsecureChannelCredentials.create() ).build(); - service = PipelineServiceGrpc.newFutureStub(channel); + + serviceFuture = PipelineServiceGrpc.newFutureStub(channel); + serviceStub = PipelineServiceGrpc.newStub(channel); int retryNo=0; final int ofRetries=10; @@ -61,8 +64,12 @@ public void connect() { } } + protected void onConnection() { + log.info("[PIPELINE GRPC] connected"); + } + boolean introduceNewConnection() throws ExecutionException, InterruptedException, TimeoutException { - uuid = service.exchangeMeta(metaNewConnection()) + uuid = serviceFuture.exchangeMeta(metaNewConnection()) .get(TIMEOUT, TimeUnit.SECONDS) .getResponse() .getStatus(); diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java index 4f3bbfc..fcc1ba2 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java @@ -1,17 +1,48 @@ package io.github.s7i.doer.pipeline.grcp; -import io.github.s7i.doer.pipeline.BlockingPipePuller; +import io.github.s7i.doer.pipeline.BlockingPipe; +import io.github.s7i.doer.pipeline.PipePuller; +import io.github.s7i.doer.pipeline.proto.MetaOp; +import io.github.s7i.doer.pipeline.proto.PipelineLoad; +import io.grpc.stub.StreamObserver; import lombok.extern.slf4j.Slf4j; @Slf4j public class GrpcInboundConnection extends GrpcConnection { - final BlockingPipePuller pipe = new BlockingPipePuller(); + final BlockingPipe pipe = new BlockingPipe(); public GrpcInboundConnection(String target) { super(target); } + @Override + public PipePuller lookupPuller() { + return pipe; + } + + @Override + protected void onConnection() { + super.onConnection(); + + var meta = MetaOp.newBuilder().build(); + serviceStub.subscribe(meta, new StreamObserver<>() { + @Override + public void onNext(PipelineLoad value) { + } + + @Override + public void onError(Throwable t) { + + } + + @Override + public void onCompleted() { + + } + }); + } + @Override public void close() throws Exception { diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java index d4af2ed..70f6e5d 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcOutboundConnection.java @@ -56,7 +56,7 @@ Optional sendLoad(Output.Load load) { .putHeaders("client.uuid", uuid) .setLoad(Any.pack(load.toRecord()))) .build(); - return Optional.ofNullable(service.publish(request).get(TIMEOUT, TimeUnit.SECONDS)); + return Optional.ofNullable(serviceFuture.publish(request).get(TIMEOUT, TimeUnit.SECONDS)); } catch (RuntimeException | TimeoutException | InterruptedException | ExecutionException e) { log.error("oops", e); } diff --git a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy index 397d1bd..a89632d 100644 --- a/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/domain/output/PipelineOutputTest.groovy @@ -1,6 +1,6 @@ package io.github.s7i.doer.domain.output -import io.github.s7i.doer.pipeline.BlockingPipePuller +import io.github.s7i.doer.pipeline.BlockingPipe import io.github.s7i.doer.pipeline.PipeConnection import spock.lang.Specification @@ -17,7 +17,7 @@ class PipelineOutputTest extends Specification { def load = Output.Load.builder() .data("test".getBytes()) .build() - BlockingPipePuller puller + BlockingPipe puller def pipeConnection = Mock(PipeConnection) { registerPuller(_) >> { args -> puller = args[0] @@ -47,7 +47,7 @@ class PipelineOutputTest extends Specification { t.setDaemon(true) return t }) - BlockingPipePuller puller + BlockingPipe puller def pipeConnection = Mock(PipeConnection) { registerPuller(_) >> { args -> puller = args[0] From 53528d37b940c850180ee01585b0b0e3af29edf9 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sat, 25 Feb 2023 18:14:10 +0100 Subject: [PATCH 31/45] [REWORK] WIP / checkpoint 3 [TAG] Pipelines #14 --- .run/Doer pipeline sink.run.xml | 16 ++++++ docs/pipeline/grpc-backend/console-sink.yml | 10 ++++ .../doer/command/util/CommandManifest.java | 8 +++ .../io/github/s7i/doer/pipeline/Pipeline.java | 55 ++++++++++++------- .../s7i/doer/pipeline/PipelineService.java | 26 ++++++++- .../doer/pipeline/grcp/GrpcConnection.java | 1 + .../pipeline/grcp/GrpcInboundConnection.java | 35 +++++++++++- .../doer/pipeline/store/PipelineStorage.java | 4 ++ 8 files changed, 130 insertions(+), 25 deletions(-) create mode 100644 .run/Doer pipeline sink.run.xml create mode 100644 docs/pipeline/grpc-backend/console-sink.yml diff --git a/.run/Doer pipeline sink.run.xml b/.run/Doer pipeline sink.run.xml new file mode 100644 index 0000000..4750996 --- /dev/null +++ b/.run/Doer pipeline sink.run.xml @@ -0,0 +1,16 @@ + + + + \ No newline at end of file diff --git a/docs/pipeline/grpc-backend/console-sink.yml b/docs/pipeline/grpc-backend/console-sink.yml new file mode 100644 index 0000000..c357ce6 --- /dev/null +++ b/docs/pipeline/grpc-backend/console-sink.yml @@ -0,0 +1,10 @@ +version: v1 +kind: sink +params: + doer.pipeline.backend: grpc + doer.pipeline.backend.target: localhost:6565 + doer.pipeline.sink: true + doer.pipeline.bind: from-ingest-to-sink + doer.pipeline.id: id-sink1 +spec: + - output: doer://console diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 61eda4f..045b806 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -4,6 +4,7 @@ import io.github.s7i.doer.command.Command; import io.github.s7i.doer.command.Ingest; import io.github.s7i.doer.command.KafkaFeeder; +import io.github.s7i.doer.command.SinkCommand; import io.github.s7i.doer.command.dump.KafkaDump; import io.github.s7i.doer.domain.ConfigProcessor; import io.github.s7i.doer.util.Banner; @@ -89,6 +90,8 @@ private Optional mapToTask(File yaml) { return Optional.of(new TaskWrapper(kind, fromManifestFile(KafkaDump.class, yaml))); case "ingest": return Optional.of(new TaskWrapper(kind, fromManifestFile(Ingest.class, yaml))); + case "sink": + return Optional.of(new TaskWrapper(kind, fromManifestFile(SinkCommand.class, yaml))); case "config": return Optional.of(() -> new ConfigProcessor(yaml).processConfig()); default: @@ -112,6 +115,11 @@ public void run() { .flatMap(Optional::stream) .collect(Collectors.toList()); + if (tasks.isEmpty()) { + log.info("Nothing to run..."); + return; + } + var pool = Executors.newFixedThreadPool(Math.min(tasks.size(), MAX_THR_COUNT), this::spawnNewThread); tasks.forEach(pool::execute); diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 00293c1..e6ef7d8 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -17,7 +17,11 @@ public class Pipeline { @Mark.Param - public static final String DOER_PIPELINE = "doer.pipeline.backend"; + public static final String DOER_PIPELINE = "doer.pipeline"; + public static final String DOER_PIPELINE_BACKEND = DOER_PIPELINE + ".backend"; + public static final String DOER_PIPELINE_BACKEND_TARGET = DOER_PIPELINE_BACKEND + ".target"; + public static final String DOER_PIPELINE_SINK = DOER_PIPELINE + ".sink"; + public static final String GRPC_BACKEND = "grpc"; public static void initFrom(Supplier> params) { var setup = params.get().entrySet() @@ -34,32 +38,41 @@ enum PipelineKind { GRPC_OUTBOUND, GRPC_INBOUND } - private Map register = new EnumMap<>(PipelineKind.class); + private final Map register = new EnumMap<>(PipelineKind.class); void init(Map params) { - var backend = requireNonNull(params.get(DOER_PIPELINE), DOER_PIPELINE).toUpperCase(); - - var kind = PipelineKind.valueOf(backend); - switch (kind) { - case GRPC_OUTBOUND: - register.put(kind, () -> { - var c = new GrpcOutboundConnection(params.get(DOER_PIPELINE +".target")); - Globals.INSTANCE.addStopHook(c::closeSafe); - return c; - }); - break; - case GRPC_INBOUND: - register.put(kind, ()-> { - var c = new GrpcInboundConnection(params.get(DOER_PIPELINE +".target")); - c.connect(); - Globals.INSTANCE.addStopHook(c::closeSafe); - return c; - }); + var backend = requireNonNull(params.get(DOER_PIPELINE_BACKEND), DOER_PIPELINE_BACKEND).toUpperCase(); + if (GRPC_BACKEND.equalsIgnoreCase(backend)) { + var kind = Boolean.parseBoolean(params.get(DOER_PIPELINE_SINK)) + ? PipelineKind.GRPC_INBOUND + : PipelineKind.GRPC_OUTBOUND; + + log.debug("kind: {}", kind); + + switch (kind) { + case GRPC_OUTBOUND: + register.put(kind, () -> { + var c = new GrpcOutboundConnection(params.get(DOER_PIPELINE_BACKEND_TARGET)); + Globals.INSTANCE.addStopHook(c::closeSafe); + return c; + }); + break; + case GRPC_INBOUND: + register.put(kind, () -> { + var c = new GrpcInboundConnection(params.get(DOER_PIPELINE_BACKEND_TARGET)); + c.setParams(params); + c.connect(); + Globals.INSTANCE.addStopHook(c::closeSafe); + return c; + }); + } } } public PipeConnection connect(String name) { - return register.get(PipelineKind.GRPC_OUTBOUND).create(); + log.debug("connect {}", name); + + return register.values().stream().findFirst().orElseThrow().create(); } public boolean isEnabled() { diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index 3247fb6..c84079e 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -54,16 +54,38 @@ public void publish(PipelinePublishRequest request, StreamObserver responseObserver) { + log.info("new connection {}", request); + var response = MetaOp.newBuilder(); if (request.getRequest().getName().equals("add-new-pipeline-client")) { - log.info("new client"); - response.setResponse(MetaOp.Response.newBuilder().setStatus(UUID.randomUUID().toString())); + + var resp = MetaOp.Response.newBuilder().setStatus(UUID.randomUUID().toString()); + + + response.setResponse(resp); + log.info("response: {}" , resp); } responseObserver.onNext(response.build()); responseObserver.onCompleted(); } + + @Override + public void subscribe(MetaOp request, StreamObserver responseObserver) { + log.info("new connection: {}", request); + + var pipe = storage.getPipe("default").orElseThrow(); + + pipe.stream(PipelineStorage.Direction.FIFO) + .map(PipelineStorage.Element::getPackage) + .peek(p -> log.info("streaming {}", p)) + .forEach(responseObserver::onNext); + + responseObserver.onCompleted(); + + log.info("stream done"); + } } /** diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java index bdcb097..d3110be 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java @@ -62,6 +62,7 @@ public void connect() { if (!success) { throw new DoerException("[PIPELINE GRPC] FAILURE"); } + onConnection(); } protected void onConnection() { diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java index fcc1ba2..e53e6b7 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java @@ -1,19 +1,31 @@ package io.github.s7i.doer.pipeline.grcp; +import com.google.protobuf.InvalidProtocolBufferException; +import io.github.s7i.doer.Doer; +import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.pipeline.BlockingPipe; import io.github.s7i.doer.pipeline.PipePuller; import io.github.s7i.doer.pipeline.proto.MetaOp; import io.github.s7i.doer.pipeline.proto.PipelineLoad; +import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import java.util.Collections; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + @Slf4j public class GrpcInboundConnection extends GrpcConnection { final BlockingPipe pipe = new BlockingPipe(); + @Setter + Map params = Collections.emptyMap(); public GrpcInboundConnection(String target) { - super(target); + super(requireNonNull(target, "target")); } @Override @@ -25,10 +37,15 @@ public PipePuller lookupPuller() { protected void onConnection() { super.onConnection(); - var meta = MetaOp.newBuilder().build(); + var meta = MetaOp.newBuilder() + .setRequest(MetaOp.Request.newBuilder() + .setName(uuid) + .putAllParameters(params)) + .build(); serviceStub.subscribe(meta, new StreamObserver<>() { @Override public void onNext(PipelineLoad value) { + unload(value); } @Override @@ -43,6 +60,20 @@ public void onCompleted() { }); } + private void unload(PipelineLoad value) { + try { + Doer.console().info("unpack {}", value); + + var rec = value.getLoad().unpack(Record.class); + pipe.offer(Output.Load.builder() + .data(rec.getData().toByteArray()) + .build()); + } catch (InvalidProtocolBufferException e) { + log.error("oops", e); + } + } + + @Override public void close() throws Exception { diff --git a/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java index 90dbe50..0378cac 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java +++ b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java @@ -162,4 +162,8 @@ public Pipe

addToPipe(String pipeName, Collection

packages) { public Pipe

addToPipe(String pipeName, P pack) { return addToPipe(pipeName, List.of(pack)); } + + public Optional> getPipe(String name) { + return Optional.ofNullable(store.get(name)); + } } From 302e873ab4042d0df3e4c0d8ff45478a4314b5e8 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 01:50:12 +0100 Subject: [PATCH 32/45] [REWORK] done / checkpoint 4 [TAG] Pipelines #14 --- .../github/s7i/doer/domain/SinkProcessor.java | 10 ++- .../s7i/doer/pipeline/PipelineService.java | 60 +++++++++++++--- .../doer/pipeline/store/ElementIterator.java | 72 +++++++++++++++++++ .../doer/pipeline/store/PipelineStorage.java | 62 +++------------- 4 files changed, 138 insertions(+), 66 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/store/ElementIterator.java diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java index 07d6d1c..81fc2db 100644 --- a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -2,6 +2,7 @@ import io.github.s7i.doer.Context; import io.github.s7i.doer.domain.output.DefaultOutputProvider; +import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.SinkManifest; import lombok.RequiredArgsConstructor; @@ -15,8 +16,13 @@ public void execute(SinkManifest.SinkSpec sinkSpec) { context.lookupPipeline().ifPresent(pipeline -> { var puller = pipeline.connect("sink").lookupPuller(); - getDefaultOutput(context).emit(puller.onNextLoad()); - puller.onAccept(); + + Output.Load load; + while (null != (load = puller.onNextLoad())) { + getDefaultOutput(context).emit(load); + + puller.onAccept(); + } }); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index c84079e..da3b9a1 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -7,8 +7,12 @@ import io.github.s7i.doer.domain.grpc.GrpcServer; import io.github.s7i.doer.pipeline.proto.*; import io.github.s7i.doer.pipeline.store.PipelineStorage; +import io.github.s7i.doer.pipeline.store.PipelineStorage.Direction; +import io.github.s7i.doer.pipeline.store.PipelineStorage.Element; import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; +import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; @@ -18,6 +22,8 @@ import java.io.IOException; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; import static java.util.Objects.nonNull; @@ -25,7 +31,7 @@ @Slf4j(topic = "doer.console") public class PipelineService implements Runnable { - @Option(names ="--port", defaultValue = "6565") + @Option(names = "--port", defaultValue = "6565") private Integer port; public static void main(String[] args) { @@ -33,9 +39,26 @@ public static void main(String[] args) { } + @Data + @RequiredArgsConstructor + static class ClientMetadata { + final String cid; + Element last; + + Element update(Element newElement) { + return last = newElement; + } + + boolean hasLast() { + return nonNull(last); + } + } + static class Handler extends PipelineServiceGrpc.PipelineServiceImplBase { PipelineStorage storage = new PipelineStorage<>(); + Map clientMetadata = new ConcurrentHashMap<>(); + @Override public void publish(PipelinePublishRequest request, StreamObserver responseObserver) { @@ -64,7 +87,7 @@ public void exchangeMeta(MetaOp request, StreamObserver responseObserver response.setResponse(resp); - log.info("response: {}" , resp); + log.info("response: {}", resp); } responseObserver.onNext(response.build()); @@ -75,12 +98,27 @@ public void exchangeMeta(MetaOp request, StreamObserver responseObserver public void subscribe(MetaOp request, StreamObserver responseObserver) { log.info("new connection: {}", request); - var pipe = storage.getPipe("default").orElseThrow(); + var clientMeta = clientMetadata.computeIfAbsent(request.getRequest().getName(), ClientMetadata::new); + + boolean streaming = true; + do { + storage.getPipe("default").ifPresent(pipe -> { + var es = clientMeta.hasLast() + ? Direction.FIFO.streamFrom(clientMeta.getLast()).skip(1) + : pipe.stream(Direction.FIFO); + + es.map(clientMeta::update) + .map(Element::getPackage) + .peek(p -> log.info("streaming {}", p)) + .forEach(responseObserver::onNext); + }); - pipe.stream(PipelineStorage.Direction.FIFO) - .map(PipelineStorage.Element::getPackage) - .peek(p -> log.info("streaming {}", p)) - .forEach(responseObserver::onNext); + try { + TimeUnit.MILLISECONDS.sleep(500); + } catch (InterruptedException e) { + streaming = false; + } + } while (streaming); responseObserver.onCompleted(); @@ -112,7 +150,7 @@ public void record( String data, @Option(names = {"-pl", "--pipeline-load"}, description = "Embed Record in PipelineLoad::Load.") boolean pipelineLoad, - @Option(names= {"-h", "--help"}, usageHelp = true) + @Option(names = {"-h", "--help"}, usageHelp = true) boolean help) { var b = Record.newBuilder(); @@ -136,9 +174,9 @@ public void record( try { byte[] bytes = pipelineLoad ? PipelineLoad.newBuilder() - .setLoad(Any.pack(b.build())) - .build() - .toByteArray() + .setLoad(Any.pack(b.build())) + .build() + .toByteArray() : b.build().toByteArray(); System.out.write(bytes); diff --git a/src/main/java/io/github/s7i/doer/pipeline/store/ElementIterator.java b/src/main/java/io/github/s7i/doer/pipeline/store/ElementIterator.java new file mode 100644 index 0000000..d64f80c --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/store/ElementIterator.java @@ -0,0 +1,72 @@ +package io.github.s7i.doer.pipeline.store; + +import java.util.Collections; +import java.util.Iterator; +import java.util.function.Supplier; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +import static java.util.Objects.*; + +public interface ElementIterator

{ + + class PipeIterator

implements Iterator> { + + PipelineStorage.Element

current; + PipelineStorage.Element

initial; + final PipelineStorage.Direction direction; + + public PipeIterator(PipelineStorage.Direction direction, PipelineStorage.Element

initial) { + requireNonNull(initial, "initial"); + requireNonNull(direction, "direction"); + + this.direction = direction; + this.initial = initial; + } + + @Override + public boolean hasNext() { + if (nonNull(initial)) { + return true; + } + + return nonNull(current) && direction.from(current).isPresent(); + } + + @Override + public PipelineStorage.Element

next() { + if (nonNull(initial)) { + + current = initial; + initial = null; + + return current; + } + + var nextStep = direction.from(requireNonNull(current, + "missing iterator element" + )); + current = nextStep.get(); + return current; + } + } + + PipelineStorage.Element

getStartElement(PipelineStorage.Direction direction); + + default Iterable> iteratorOf(PipelineStorage.Direction direction, PipelineStorage.Element

start) { + + if (isNull(start)) { + return Collections::emptyIterator; + } + + Supplier>> prov = () -> + new PipeIterator<>(direction, start); + + return prov::get; + } + + default Stream> stream(PipelineStorage.Direction direction) { + var i = iteratorOf(direction, getStartElement(direction)); + return StreamSupport.stream(i.spliterator(), false); + } +} diff --git a/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java index 0378cac..1f9d4a3 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java +++ b/src/main/java/io/github/s7i/doer/pipeline/store/PipelineStorage.java @@ -6,9 +6,7 @@ import lombok.experimental.Accessors; import java.util.*; -import java.util.function.Supplier; import java.util.stream.Stream; -import java.util.stream.StreamSupport; import static java.util.Objects.*; @@ -27,56 +25,21 @@ Optional> from(Element e) { } throw new IllegalStateException(); } + + public Stream> streamFrom(Element e) { + ElementIterator ei = direction -> e; + return ei.stream(this); + } } public interface OffsetSequence { Long nextOffset(); } - static class PipeIterator

implements Iterator> { - - Element

current; - Element

initial; - final Direction direction; - - PipeIterator(Direction direction, Element

initial) { - requireNonNull(initial, "initial"); - requireNonNull(direction, "direction"); - - this.direction = direction; - this.initial = initial; - } - - @Override - public boolean hasNext() { - if (nonNull(initial)) { - return true; - } - - return nonNull(current) && direction.from(current).isPresent(); - } - - @Override - public Element

next() { - if (nonNull(initial)) { - - current = initial; - initial = null; - - return current; - } - - var nextStep = direction.from(requireNonNull(current, - "missing iterator element" - )); - current = nextStep.get(); - return current; - } - } @Data @Accessors(fluent = true) - public static class Pipe

{ + public static class Pipe

implements ElementIterator

{ Element

tail; Element

head; @@ -94,17 +57,10 @@ public void putElementWithPackage(P pack, OffsetSequence oseq) { head = el; } } - public Iterable> iterator(Direction direction) { - Supplier>> prov = () -> new PipeIterator<>( - direction, - direction == Direction.FIFO ? head : tail - ); - return prov::get; - } - - public Stream> stream(Direction direction) { - return StreamSupport.stream(iterator(direction).spliterator(), false); + @Override + public Element

getStartElement(Direction direction) { + return direction == Direction.FIFO ? head : tail; } } From bbdf231ac461ade0f5e9eabce242a3cdb67d2f79 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 10:49:48 +0100 Subject: [PATCH 33/45] [REWORK] test fix [TAG] Pipelines #14 --- src/main/java/io/github/s7i/doer/pipeline/Pipeline.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index e6ef7d8..27aeb8e 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -11,7 +11,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static java.util.Objects.requireNonNull; +import static java.util.Objects.nonNull; @Slf4j public class Pipeline { @@ -26,7 +26,7 @@ public class Pipeline { public static void initFrom(Supplier> params) { var setup = params.get().entrySet() .stream() - .filter(es -> es.getKey().startsWith(DOER_PIPELINE)) + .filter(es -> es.getKey().startsWith(DOER_PIPELINE) && nonNull(es.getValue())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); if (!setup.isEmpty()) { @@ -41,7 +41,7 @@ enum PipelineKind { private final Map register = new EnumMap<>(PipelineKind.class); void init(Map params) { - var backend = requireNonNull(params.get(DOER_PIPELINE_BACKEND), DOER_PIPELINE_BACKEND).toUpperCase(); + var backend = params.get(DOER_PIPELINE_BACKEND); if (GRPC_BACKEND.equalsIgnoreCase(backend)) { var kind = Boolean.parseBoolean(params.get(DOER_PIPELINE_SINK)) ? PipelineKind.GRPC_INBOUND From c574e8463a188f7c95477ce99c0abc8f53e2a546 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 11:47:03 +0100 Subject: [PATCH 34/45] [REWORK] extracted mappers [TAG] Pipelines #14 --- .../io/github/s7i/doer/domain/Mappers.java | 51 +++++++++++++++++++ .../doer/domain/ingest/IngestProcessor.java | 29 +++++++++-- .../github/s7i/doer/domain/output/Output.java | 29 ++--------- .../pipeline/grcp/GrpcInboundConnection.java | 15 +++--- 4 files changed, 87 insertions(+), 37 deletions(-) create mode 100644 src/main/java/io/github/s7i/doer/domain/Mappers.java diff --git a/src/main/java/io/github/s7i/doer/domain/Mappers.java b/src/main/java/io/github/s7i/doer/domain/Mappers.java new file mode 100644 index 0000000..bb22716 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/domain/Mappers.java @@ -0,0 +1,51 @@ +package io.github.s7i.doer.domain; + +import com.google.protobuf.ByteString; +import com.google.protobuf.BytesValue; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.StringValue; +import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.pipeline.proto.PipelineLoad; +import io.github.s7i.doer.proto.Record; +import lombok.experimental.UtilityClass; + +import static java.util.Objects.nonNull; + +@UtilityClass +public class Mappers { + + public static Output.Load mapFrom(PipelineLoad value) throws InvalidProtocolBufferException { + var rec = value.getLoad().unpack(Record.class); + var load = Output.Load.builder() + .data(rec.getData().toByteArray()) + .build(); + + return load; + + } + + public static Record mapFrom(Output.Load load) { + var b = Record.newBuilder(); + if (nonNull(load.getResource())) { + b.setResource(StringValue.of(load.getResource())); + } + if (nonNull(load.getKey())) { + b.setKey(StringValue.of(load.getKey())); + } + if (nonNull(load.getMeta())) { + b.putAllMeta(load.getMeta()); + } + var data = load.getData(); + if (nonNull(data) && data.length > 0) { + b.setData(BytesValue.of(ByteString.copyFrom(data))); + } + return b.build(); + } + + public static Output.Load mapFrom(String resource, byte[] data) { + return Output.Load.builder() + .resource(resource) + .data(data) + .build(); + } +} diff --git a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java index 59f542d..004af80 100644 --- a/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/ingest/IngestProcessor.java @@ -5,27 +5,46 @@ import io.github.s7i.doer.domain.output.DefaultOutputProvider; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.ingest.IngestRecordManifest; +import io.github.s7i.doer.util.PropertyResolver; import lombok.RequiredArgsConstructor; import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Optional; @RequiredArgsConstructor public class IngestProcessor implements DefaultOutputProvider { private final Context context; + private PropertyResolver propertyResolver; public void process(IngestRecordManifest manifest) { + propertyResolver = context.getPropertyResolver(); + try (var output = getDefaultOutput(context)) { manifest.getRecords().stream() - .map(rec -> Output.Load.builder() - .data(context.getPropertyResolver() - .resolve(rec.getRecord()) - .getBytes(StandardCharsets.UTF_8)) - .build()) + .map(this::resolveRecord) .forEach(output::emit); } catch (Exception e) { throw new DoerException(e); } } + + private Output.Load resolveRecord(IngestRecordManifest.Record record) { + final var bld = Output.Load.builder(); + + Optional.ofNullable(record.getRecord()).ifPresent(data -> + bld.data(propertyResolver + .resolve(record.getRecord()) + .getBytes(StandardCharsets.UTF_8)) + ); + + Optional.ofNullable(record.getKey()).ifPresent(key -> + bld.key(propertyResolver.resolve(record.getKey())) + ); + bld.meta(Map.of("doer.record.type", "ingest-record")); + + return bld.build(); + } } diff --git a/src/main/java/io/github/s7i/doer/domain/output/Output.java b/src/main/java/io/github/s7i/doer/domain/output/Output.java index 6a75eaa..bd10193 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/Output.java +++ b/src/main/java/io/github/s7i/doer/domain/output/Output.java @@ -1,16 +1,13 @@ package io.github.s7i.doer.domain.output; -import static java.util.Objects.nonNull; - -import com.google.protobuf.ByteString; -import com.google.protobuf.BytesValue; -import com.google.protobuf.StringValue; +import io.github.s7i.doer.domain.Mappers; import io.github.s7i.doer.proto.Record; -import java.util.Map; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import java.util.Map; + public interface Output extends AutoCloseable { @Builder @@ -28,30 +25,14 @@ public String dataAsString() { } public Record toRecord() { - var b = Record.newBuilder(); - if (nonNull(resource)) { - b.setResource(StringValue.of(resource)); - } - if (nonNull(key)) { - b.setKey(StringValue.of(key)); - } - if (nonNull(meta)) { - b.putAllMeta(meta); - } - if (nonNull(data) && data.length > 0) { - b.setData(BytesValue.of(ByteString.copyFrom(data))); - } - return b.build(); + return Mappers.mapFrom(this); } } void open(); default void emit(String resource, byte[] data) { - emit(Load.builder() - .resource(resource) - .data(data) - .build()); + emit(Mappers.mapFrom(resource, data)); } void emit(Load load); diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java index e53e6b7..be65510 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java @@ -2,12 +2,11 @@ import com.google.protobuf.InvalidProtocolBufferException; import io.github.s7i.doer.Doer; -import io.github.s7i.doer.domain.output.Output; +import io.github.s7i.doer.domain.Mappers; import io.github.s7i.doer.pipeline.BlockingPipe; import io.github.s7i.doer.pipeline.PipePuller; import io.github.s7i.doer.pipeline.proto.MetaOp; import io.github.s7i.doer.pipeline.proto.PipelineLoad; -import io.github.s7i.doer.proto.Record; import io.grpc.stub.StreamObserver; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -50,12 +49,12 @@ public void onNext(PipelineLoad value) { @Override public void onError(Throwable t) { - + log.error("oops", t); } @Override public void onCompleted() { - + log.debug("subscribe done"); } }); } @@ -64,10 +63,10 @@ private void unload(PipelineLoad value) { try { Doer.console().info("unpack {}", value); - var rec = value.getLoad().unpack(Record.class); - pipe.offer(Output.Load.builder() - .data(rec.getData().toByteArray()) - .build()); + if (value.hasLoad()) { + var load = Mappers.mapFrom(value); + pipe.offer(load); + } } catch (InvalidProtocolBufferException e) { log.error("oops", e); } From 05a86088d2e2d88030a5170aea20c8295a01e010 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 13:34:06 +0100 Subject: [PATCH 35/45] [ADD] protocol stub [TAG] Pipelines #14 --- .../io/github/s7i/doer/pipeline/PipelineService.java | 2 +- .../java/io/github/s7i/doer/pipeline/Protocol.java | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/github/s7i/doer/pipeline/Protocol.java diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index da3b9a1..69594ee 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -81,7 +81,7 @@ public void exchangeMeta(MetaOp request, StreamObserver responseObserver var response = MetaOp.newBuilder(); - if (request.getRequest().getName().equals("add-new-pipeline-client")) { + if (request.getRequest().getName().equals(Protocol.OP_ADD_NEW)) { var resp = MetaOp.Response.newBuilder().setStatus(UUID.randomUUID().toString()); diff --git a/src/main/java/io/github/s7i/doer/pipeline/Protocol.java b/src/main/java/io/github/s7i/doer/pipeline/Protocol.java new file mode 100644 index 0000000..3be2680 --- /dev/null +++ b/src/main/java/io/github/s7i/doer/pipeline/Protocol.java @@ -0,0 +1,12 @@ +package io.github.s7i.doer.pipeline; + +import io.github.s7i.doer.pipeline.proto.MetaOp; + +public class Protocol { + + public interface Operation { + MetaOp perform(MetaOp action); + } + + public static final String OP_ADD_NEW = "add-new"; +} From 6928de85c953dfb31a7a251cf4a07880ece110b0 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 13:35:09 +0100 Subject: [PATCH 36/45] [REWORK] sink processor improvements [TAG] Pipelines #14 --- .../s7i/doer/command/CommandWithContext.java | 4 +- .../github/s7i/doer/command/SinkCommand.java | 6 +- .../github/s7i/doer/domain/SinkProcessor.java | 91 +++++++++++++++++-- .../s7i/doer/domain/output/OutputBuilder.java | 2 +- .../s7i/doer/manifest/SinkManifest.java | 3 +- .../io/github/s7i/doer/pipeline/Pipeline.java | 4 +- .../doer/pipeline/grcp/GrpcConnection.java | 3 +- 7 files changed, 99 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/CommandWithContext.java b/src/main/java/io/github/s7i/doer/command/CommandWithContext.java index aa7697a..d8c61ff 100644 --- a/src/main/java/io/github/s7i/doer/command/CommandWithContext.java +++ b/src/main/java/io/github/s7i/doer/command/CommandWithContext.java @@ -3,13 +3,15 @@ import io.github.s7i.doer.Context; import io.github.s7i.doer.config.Base; +import static java.util.Objects.requireNonNull; + public abstract class CommandWithContext extends ManifestFileCommand { protected abstract Class manifestClass(); @Override public void onExecuteCommand() { - var manifest = parseYaml(manifestClass()); + var manifest = requireNonNull(parseYaml(manifestClass()), "manifest definition"); var ctx = new Context.Initializer(Context.InitialParameters.builder() .workDir(yaml.toPath().toAbsolutePath().getParent()) diff --git a/src/main/java/io/github/s7i/doer/command/SinkCommand.java b/src/main/java/io/github/s7i/doer/command/SinkCommand.java index e445ac1..1cf958a 100644 --- a/src/main/java/io/github/s7i/doer/command/SinkCommand.java +++ b/src/main/java/io/github/s7i/doer/command/SinkCommand.java @@ -6,6 +6,9 @@ import picocli.CommandLine; import java.io.File; +import java.util.Collections; + +import static java.util.Objects.isNull; @CommandLine.Command(name = "sink") public class SinkCommand extends CommandWithContext { @@ -22,6 +25,7 @@ protected Class manifestClass() { @Override public void onExecuteCommand(Context context, SinkManifest manifest) { - new SinkProcessor(context).execute(manifest.getSpec().stream().findFirst().orElseThrow()); + var list = manifest.getSpec(); + new SinkProcessor(context).execute(isNull(list) ? Collections.emptyList() : list); } } diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java index 81fc2db..4dcff98 100644 --- a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -1,30 +1,109 @@ package io.github.s7i.doer.domain; import io.github.s7i.doer.Context; +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.output.DefaultOutputProvider; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.SinkManifest; +import io.github.s7i.doer.pipeline.PipePuller; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static java.util.Objects.nonNull; @RequiredArgsConstructor +@Slf4j public class SinkProcessor implements DefaultOutputProvider { final Context context; + ExecutorService executorService; + CountDownLatch taskToComplete; - public void execute(SinkManifest.SinkSpec sinkSpec) { + public void execute(List specList) { context.lookupPipeline().ifPresent(pipeline -> { - var puller = pipeline.connect("sink").lookupPuller(); + var puller = pipeline.connect().lookupPuller(); - Output.Load load; - while (null != (load = puller.onNextLoad())) { - getDefaultOutput(context).emit(load); + var enabled = specList.stream() + .filter(SinkManifest.SinkSpec::isEnabled) + .collect(Collectors.toList()); - puller.onAccept(); + if (!enabled.isEmpty()) { + enableOutputs(enabled, puller); + } else { + drainPipeline(puller); } }); + if (nonNull(taskToComplete)) { + log.debug("Awaiting to complete all tasks..."); + + try { + taskToComplete.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DoerException(e); + } + try { + executorService.shutdown(); + if (!executorService.awaitTermination(30, TimeUnit.SECONDS)) { + log.warn("Some timeouts..."); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new DoerException(e); + } + log.debug("All tasks done."); + } + } + + private void enableOutputs(List specList, PipePuller puller) { + var size = specList.size(); + + log.debug("Number of output tasks: {}", size); + + taskToComplete = new CountDownLatch(size); + + var tg = new ThreadGroup("doer-output-workers"); + executorService = Executors.newFixedThreadPool(size, runnable -> + new Thread(tg, runnable, "output-worker")); + + specList.stream() + .map(s -> toRunnable(s, puller)) + .forEach(executorService::submit); + } + + private Runnable toRunnable(SinkManifest.SinkSpec spec, PipePuller puller) { + return () -> { + loopOutput(context.buildOutput(spec::getOutput), puller); + taskToComplete.countDown(); + log.debug("task completed {}", Thread.currentThread().getName()); + }; + } + + private void drainPipeline(PipePuller puller) { + log.debug("Using default output."); + + var out = getDefaultOutput(context); + loopOutput(out, puller); + } + + public static void loopOutput(Output out, PipePuller puller) { + Output.Load load; + while (null != (load = puller.onNextLoad())) { + out.emit(load); + puller.onAccept(); + } + LoggerFactory.getLogger(SinkManifest.class).debug("output loop ends."); } } diff --git a/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java b/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java index 532b779..b52f984 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java +++ b/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java @@ -19,7 +19,7 @@ public Output build(OutputProvider outputProvider) { FileOutputCreator foc = () -> context.getBaseDir().resolve(outputProvider.getOutput()); HttpOutputCreator http = outputProvider::getOutput; KafkaOutputCreator kafka = new KafkaUri(outputProvider, context); - PipelineOutputCreator pipeline = () -> Globals.INSTANCE.getPipeline().connect(outputProvider.getOutput()); + PipelineOutputCreator pipeline = () -> Globals.INSTANCE.getPipeline().connect(); final var factory = context.getOutputFactory(); diff --git a/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java b/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java index 3a70962..e00483e 100644 --- a/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java +++ b/src/main/java/io/github/s7i/doer/manifest/SinkManifest.java @@ -11,7 +11,8 @@ public class SinkManifest extends Base { @Data public static class SinkSpec { - String output; + String output = ""; + boolean enabled = true; } @JsonProperty("spec") diff --git a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java index 27aeb8e..c955d9a 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java +++ b/src/main/java/io/github/s7i/doer/pipeline/Pipeline.java @@ -69,9 +69,7 @@ void init(Map params) { } } - public PipeConnection connect(String name) { - log.debug("connect {}", name); - + public PipeConnection connect() { return register.values().stream().findFirst().orElseThrow().create(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java index d3110be..5502223 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcConnection.java @@ -2,6 +2,7 @@ import io.github.s7i.doer.DoerException; import io.github.s7i.doer.pipeline.PipeConnection; +import io.github.s7i.doer.pipeline.Protocol; import io.github.s7i.doer.pipeline.proto.MetaOp; import io.github.s7i.doer.pipeline.proto.PipelineServiceGrpc; import io.grpc.Grpc; @@ -81,7 +82,7 @@ boolean introduceNewConnection() throws ExecutionException, InterruptedException static MetaOp metaNewConnection() { return MetaOp.newBuilder() - .setRequest(MetaOp.Request.newBuilder().setName("add-new-pipeline-client")) + .setRequest(MetaOp.Request.newBuilder().setName(Protocol.OP_ADD_NEW)) .build(); } From b33877fced00e7c9b96dfd06c4806c5ca72136d4 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 15:05:41 +0100 Subject: [PATCH 37/45] [REWORK] shadowJar issue [TAG] Pipelines #14 --- build.gradle | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.gradle b/build.gradle index f04fc00..13563a7 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,11 @@ application { mainClass.set('io.github.s7i.doer.Doer') } +shadowJar { + // https://github.com/grpc/grpc-java/issues/5493#issuecomment-478500418 + mergeServiceFiles() +} + tasks.withType(JavaCompile) { options.compilerArgs << '-Xlint:unchecked' options.deprecation = true From 401d20ec9eb5da8a7cc8441841890560254f4187 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 15:06:42 +0100 Subject: [PATCH 38/45] [REWORK] invalid flow handling improvements [TAG] Pipelines #14 --- .../s7i/doer/command/util/CommandManifest.java | 9 +++++++-- .../github/s7i/doer/domain/SinkProcessor.java | 17 ++++++++++++++--- .../s7i/doer/pipeline/PipelineService.java | 12 +++++++++--- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 045b806..9de4257 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -8,6 +8,7 @@ import io.github.s7i.doer.command.dump.KafkaDump; import io.github.s7i.doer.domain.ConfigProcessor; import io.github.s7i.doer.util.Banner; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import picocli.CommandLine; @@ -57,6 +58,8 @@ static class TaskWrapper implements Runnable { final String name; final Command coreTask; Duration duration; + @Getter + boolean failed; @Override public void run() { @@ -66,14 +69,16 @@ public void run() { try { coreTask.call(); } catch (Exception e) { - log.error("fatal error", e); + log.error("[TASK FAILURE]", e); + failed = true; } } finally { duration = Duration.between(begin, Instant.now()); } } + public String getSummary() { - return String.format("Task %s ends in %s", name, duration); + return String.format("%sTask %s ends in %s", failed ? "[FAILED] " : "", name, duration); } } diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java index 4dcff98..b0bf37e 100644 --- a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -28,6 +28,8 @@ public class SinkProcessor implements DefaultOutputProvider { ExecutorService executorService; CountDownLatch taskToComplete; + transient int failureCount; + public void execute(List specList) { context.lookupPipeline().ifPresent(pipeline -> { @@ -63,6 +65,9 @@ public void execute(List specList) { } log.debug("All tasks done."); } + if (failureCount > 0) { + throw new DoerException("Some failures during executions."); + } } private void enableOutputs(List specList, PipePuller puller) { @@ -83,9 +88,15 @@ private void enableOutputs(List specList, PipePuller pull private Runnable toRunnable(SinkManifest.SinkSpec spec, PipePuller puller) { return () -> { - loopOutput(context.buildOutput(spec::getOutput), puller); - taskToComplete.countDown(); - log.debug("task completed {}", Thread.currentThread().getName()); + try { + loopOutput(context.buildOutput(spec::getOutput), puller); + } catch (RuntimeException r) { + log.error("worker task issue", r); + failureCount++; + } finally { + log.debug("task completed {}", Thread.currentThread().getName()); + taskToComplete.countDown(); + } }; } diff --git a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java index 69594ee..f021f53 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java +++ b/src/main/java/io/github/s7i/doer/pipeline/PipelineService.java @@ -10,6 +10,7 @@ import io.github.s7i.doer.pipeline.store.PipelineStorage.Direction; import io.github.s7i.doer.pipeline.store.PipelineStorage.Element; import io.github.s7i.doer.proto.Record; +import io.github.s7i.doer.util.Banner; import io.grpc.stub.StreamObserver; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -22,6 +23,7 @@ import java.io.IOException; import java.util.Map; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -29,13 +31,13 @@ @Command(name = "pipeline", description = "Pipeline Backend Service.") @Slf4j(topic = "doer.console") -public class PipelineService implements Runnable { +public class PipelineService implements Callable, Banner { @Option(names = "--port", defaultValue = "6565") private Integer port; public static void main(String[] args) { - new CommandLine(PipelineService.class).execute(args); + System.exit(new CommandLine(PipelineService.class).execute(args)); } @@ -188,9 +190,13 @@ public void record( @SneakyThrows @Override - public void run() { + public Integer call() { + printBanner(); + var service = new Handler(); new GrpcServer(port, service) .startServer(); + + return 0; } } From 755d786e058f82616cb8acf289d6aa5d35bbe5cd Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 16:26:24 +0100 Subject: [PATCH 39/45] [REWORK] kafka output enabled [TAG] Pipelines #14 --- docs/pipeline/grpc-backend/console-sink.yml | 6 +++ docs/pipeline/grpc-backend/readme.md | 17 +++++++++ src/main/java/io/github/s7i/doer/Context.java | 6 +++ .../doer/command/util/CommandManifest.java | 11 ++++-- .../github/s7i/doer/domain/SinkProcessor.java | 9 ++++- .../doer/domain/kafka/output/KafkaOutput.java | 37 +++++++++++++++++-- .../s7i/doer/domain/output/OutputBuilder.java | 14 +++++-- 7 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 docs/pipeline/grpc-backend/readme.md diff --git a/docs/pipeline/grpc-backend/console-sink.yml b/docs/pipeline/grpc-backend/console-sink.yml index c357ce6..edbd90d 100644 --- a/docs/pipeline/grpc-backend/console-sink.yml +++ b/docs/pipeline/grpc-backend/console-sink.yml @@ -6,5 +6,11 @@ params: doer.pipeline.sink: true doer.pipeline.bind: from-ingest-to-sink doer.pipeline.id: id-sink1 + mykaf: |+ + bootstrap.servers=localhost:9092 + client.id=doer.sink spec: - output: doer://console +# enabled: false + - output: kafka://mykaf/test-sink123 +# enable: false diff --git a/docs/pipeline/grpc-backend/readme.md b/docs/pipeline/grpc-backend/readme.md new file mode 100644 index 0000000..8c06331 --- /dev/null +++ b/docs/pipeline/grpc-backend/readme.md @@ -0,0 +1,17 @@ +### Running from the console + +1. Sink: + + ```bash + doer console-sink.yml + + ``` + +2. Service: + ```bash + doer pipeline + ``` +3. Records ingest + ```bash + doer ingest.yml + ``` diff --git a/src/main/java/io/github/s7i/doer/Context.java b/src/main/java/io/github/s7i/doer/Context.java index e255fbe..bd72ca9 100644 --- a/src/main/java/io/github/s7i/doer/Context.java +++ b/src/main/java/io/github/s7i/doer/Context.java @@ -14,6 +14,7 @@ import java.nio.file.Path; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -95,4 +96,9 @@ default Optional lookupPipeline() { } return Optional.empty(); } + + default void shareParams(Map otherParams) { + var copy = new HashMap<>(otherParams); + Globals.INSTANCE.getScope().setParams(() -> copy); + } } diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 9de4257..4fcc1d9 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -1,5 +1,6 @@ package io.github.s7i.doer.command.util; +import io.github.s7i.doer.Doer; import io.github.s7i.doer.Globals; import io.github.s7i.doer.command.Command; import io.github.s7i.doer.command.Ingest; @@ -22,6 +23,7 @@ import java.time.Instant; import java.util.List; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -37,7 +39,7 @@ description = "Parsing command manifest yaml file." ) @Slf4j(topic = DOER_CONSOLE) -public class CommandManifest implements Runnable, Banner { +public class CommandManifest implements Callable, Banner { public static final int MAX_THR_COUNT = 20; @Parameters(arity = "1..*") @@ -67,7 +69,7 @@ public void run() { var begin = Instant.now(); try { try { - coreTask.call(); + failed = 0 != coreTask.call(); } catch (Exception e) { log.error("[TASK FAILURE]", e); failed = true; @@ -111,7 +113,7 @@ private Optional mapToTask(File yaml) { } @Override - public void run() { + public Integer call() { printBanner(); requireNonNull(yamls, "manifest file set..."); @@ -122,7 +124,7 @@ public void run() { if (tasks.isEmpty()) { log.info("Nothing to run..."); - return; + return Doer.EC_INVALID_USAGE; } var pool = Executors.newFixedThreadPool(Math.min(tasks.size(), MAX_THR_COUNT), this::spawnNewThread); @@ -152,5 +154,6 @@ public void run() { log.error("oops", e); Thread.currentThread().interrupt(); } + return 0; } } diff --git a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java index b0bf37e..8bee1d6 100644 --- a/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java +++ b/src/main/java/io/github/s7i/doer/domain/SinkProcessor.java @@ -2,6 +2,7 @@ import io.github.s7i.doer.Context; import io.github.s7i.doer.DoerException; +import io.github.s7i.doer.Globals; import io.github.s7i.doer.domain.output.DefaultOutputProvider; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.manifest.SinkManifest; @@ -11,6 +12,7 @@ import org.slf4j.LoggerFactory; import java.util.List; +import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -71,6 +73,7 @@ public void execute(List specList) { } private void enableOutputs(List specList, PipePuller puller) { + var initial = context.getParams(); var size = specList.size(); log.debug("Number of output tasks: {}", size); @@ -82,13 +85,15 @@ private void enableOutputs(List specList, PipePuller pull new Thread(tg, runnable, "output-worker")); specList.stream() - .map(s -> toRunnable(s, puller)) + .map(s -> toRunnable(s, puller, initial)) .forEach(executorService::submit); } - private Runnable toRunnable(SinkManifest.SinkSpec spec, PipePuller puller) { + private Runnable toRunnable(SinkManifest.SinkSpec spec, PipePuller puller, Map param) { return () -> { try { + Globals.INSTANCE.shareParams(param); + loopOutput(context.buildOutput(spec::getOutput), puller); } catch (RuntimeException r) { log.error("worker task issue", r); diff --git a/src/main/java/io/github/s7i/doer/domain/kafka/output/KafkaOutput.java b/src/main/java/io/github/s7i/doer/domain/kafka/output/KafkaOutput.java index 9382700..7d820d5 100644 --- a/src/main/java/io/github/s7i/doer/domain/kafka/output/KafkaOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/kafka/output/KafkaOutput.java @@ -1,38 +1,65 @@ package io.github.s7i.doer.domain.kafka.output; -import static java.util.Objects.nonNull; - +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.kafka.KafkaConfig; import io.github.s7i.doer.domain.kafka.KafkaFactory; import io.github.s7i.doer.domain.output.Output; -import java.nio.charset.StandardCharsets; import lombok.Builder; +import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.header.internals.RecordHeader; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static java.util.Objects.nonNull; +import static java.util.Objects.requireNonNull; + @Builder +@Slf4j public class KafkaOutput implements Output { + public static final int RECORD_TIMEOUT_SEC = 30; final KafkaFactory kafkaFactory; final KafkaConfig config; final boolean userTracing; final TopicPartition topic; Producer producer; + boolean closed; + @Override public void open() { + if (nonNull(producer)) { + return; + } producer = kafkaFactory.getProducerFactory().createProducer(config, userTracing); } @Override public void emit(Load load) { + if (closed) { + throw new DoerException("closed"); + } + requireNonNull(producer, "producer"); var record = new ProducerRecord<>(topic.topic(), load.getKey(), load.getData()); if (nonNull(load.getMeta())) { load.getMeta().forEach((k, v) -> record.headers().add(new RecordHeader(k, v.getBytes(StandardCharsets.UTF_8)))); } - producer.send(record); + log.debug("sending record..."); + try { + producer.send(record, (m, t) -> log.debug("rec-callback: {}", m, t)) + .get(RECORD_TIMEOUT_SEC, TimeUnit.SECONDS); + } catch (InterruptedException itr) { + Thread.currentThread().interrupt(); + throw new DoerException("interrupted thread"); + } catch (ExecutionException | TimeoutException e) { + throw new DoerException(e); + } } @Override @@ -40,5 +67,7 @@ public void close() throws Exception { if (nonNull(producer)) { producer.close(); } + closed = true; + log.debug("closed"); } } diff --git a/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java b/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java index b52f984..fba3d01 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java +++ b/src/main/java/io/github/s7i/doer/domain/output/OutputBuilder.java @@ -9,14 +9,18 @@ import io.github.s7i.doer.domain.output.creator.PipelineOutputCreator; import lombok.Setter; import lombok.experimental.Accessors; +import lombok.extern.slf4j.Slf4j; @Accessors(fluent = true) @Setter +@Slf4j public class OutputBuilder { Context context; public Output build(OutputProvider outputProvider) { - FileOutputCreator foc = () -> context.getBaseDir().resolve(outputProvider.getOutput()); + var def = outputProvider.getOutput(); + + FileOutputCreator foc = () -> context.getBaseDir().resolve(def); HttpOutputCreator http = outputProvider::getOutput; KafkaOutputCreator kafka = new KafkaUri(outputProvider, context); PipelineOutputCreator pipeline = () -> Globals.INSTANCE.getPipeline().connect(); @@ -28,8 +32,12 @@ public Output build(OutputProvider outputProvider) { factory.register(OutputKind.KAFKA, kafka); factory.register(OutputKind.PIPELINE, pipeline); - return factory.resolve(new UriResolver(outputProvider.getOutput())) - .orElseThrow(); + var output = factory.resolve(new UriResolver(def)).orElseThrow(); + log.debug("resolved output {}", output); + //TODO: auto open make configurable + output.open(); + + return output; } } From b49721ccca7810cef46fdc82b0c075c76c0f14d3 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 16:38:29 +0100 Subject: [PATCH 40/45] [REWORK] mock adjustments [TAG] Pipelines #14 --- .../io/github/s7i/doer/command/dump/KafkaDumpTest.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/groovy/io/github/s7i/doer/command/dump/KafkaDumpTest.groovy b/src/test/groovy/io/github/s7i/doer/command/dump/KafkaDumpTest.groovy index 7f11f04..761b473 100644 --- a/src/test/groovy/io/github/s7i/doer/command/dump/KafkaDumpTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/command/dump/KafkaDumpTest.groovy @@ -15,6 +15,7 @@ import spock.lang.Specification import java.time.LocalDateTime import java.time.ZoneOffset +import java.util.concurrent.Future class KafkaDumpTest extends Specification { @@ -144,7 +145,7 @@ class KafkaDumpTest extends Specification { def prodFactory = Mock(KafkaProducerFactory) { 1 * createProducer(_, _) >> Mock(Producer) { - 10 * send(_) + 10 * send(_, _) >> Mock(Future) } } From 2426c8acf30eaaf9ae396d6dcd713485c80a574a Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 22:30:31 +0100 Subject: [PATCH 41/45] [REWORK] better test [TAG] Pipelines #14 --- .../ingest/record/IngestRecordTest.groovy | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy index e85770c..f98cc0b 100644 --- a/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy @@ -1,25 +1,39 @@ package io.github.s7i.doer.ingest.record -import io.github.s7i.doer.Globals +import io.github.s7i.doer.Context import io.github.s7i.doer.command.YamlParser import io.github.s7i.doer.domain.ingest.IngestProcessor +import io.github.s7i.doer.domain.output.Output import io.github.s7i.doer.manifest.ingest.IngestRecordManifest +import io.github.s7i.doer.util.PropertyResolver import spock.lang.Specification import java.nio.file.Path class IngestRecordTest extends Specification { - def "Parse Manifest" () { + def "Parse Manifest"() { given: YamlParser yml = { Path.of("src/test/resources/ingest/ingest-record.yml").toFile() } def manifest = yml.parseYaml(IngestRecordManifest.class) - new IngestProcessor(Globals.INSTANCE).process(manifest) + def records = [] + def context = Mock(Context) { + getParams() >> ["doer.output": "mock-me"] + getPropertyResolver() >> new PropertyResolver() + buildOutput(_) >> Mock(Output) { + 3 * emit(_) >> { + records << it[0] + } + } + } + + new IngestProcessor(context).process(manifest) expect: manifest.getKind() == 'ingest' + records.size() == 3 } From 08434eb8100e2d857c29085abf6c9ed68b8306aa Mon Sep 17 00:00:00 2001 From: nefro85 Date: Sun, 26 Feb 2023 22:30:56 +0100 Subject: [PATCH 42/45] [REWORK] improvements [TAG] Pipelines #14 --- .../github/s7i/doer/command/util/CommandManifest.java | 2 +- .../io/github/s7i/doer/domain/output/HttpOutput.java | 10 ++++++++-- .../github/s7i/doer/domain/output/PipelineOutput.java | 4 ++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index 4fcc1d9..bffd883 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -138,7 +138,7 @@ public Integer call() { log.info("Summary:"); var sum = tasks.stream() - .filter(t -> t instanceof TaskWrapper) + .filter(TaskWrapper.class::isInstance) .mapToLong(t -> { var task = (TaskWrapper) t; log.info(task.getSummary()); diff --git a/src/main/java/io/github/s7i/doer/domain/output/HttpOutput.java b/src/main/java/io/github/s7i/doer/domain/output/HttpOutput.java index c0d84ea..d6ef071 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/HttpOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/HttpOutput.java @@ -1,5 +1,8 @@ package io.github.s7i.doer.domain.output; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -7,8 +10,8 @@ import java.net.http.HttpRequest.BodyPublishers; import java.net.http.HttpResponse.BodyHandlers; import java.time.Duration; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; + +import static java.util.Objects.nonNull; @Slf4j @RequiredArgsConstructor @@ -19,6 +22,9 @@ public class HttpOutput implements Output { @Override public void open() { + if (nonNull(httpClient)) { + return; + } httpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(15)) .build(); diff --git a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java index 5c49ff2..f0ad9fd 100644 --- a/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java +++ b/src/main/java/io/github/s7i/doer/domain/output/PipelineOutput.java @@ -4,6 +4,7 @@ import io.github.s7i.doer.pipeline.PipeConnection; import lombok.RequiredArgsConstructor; +import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; @RequiredArgsConstructor @@ -15,6 +16,9 @@ public class PipelineOutput implements Output { @Override public void open() { + if (nonNull(pusher)) { + return; + } pusher = new BlockingPipe(); pipeConnection.registerPuller(pusher); } From 17665c3afdb1142b6ff2e1f4e83ab73fc8124c7f Mon Sep 17 00:00:00 2001 From: nefro85 Date: Mon, 27 Feb 2023 00:17:32 +0100 Subject: [PATCH 43/45] [REWORK] better tests [TAG] Pipelines #14 --- .../doer/command/util/CommandManifest.java | 12 +++++++- .../io/github/s7i/doer/domain/Mappers.java | 30 ++++++++++++++++--- .../pipeline/grcp/GrpcInboundConnection.java | 5 ++-- .../ingest/record/IngestRecordTest.groovy | 3 +- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java index bffd883..b2cd808 100644 --- a/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java +++ b/src/main/java/io/github/s7i/doer/command/util/CommandManifest.java @@ -132,7 +132,9 @@ public Integer call() { try { pool.shutdown(); - pool.awaitTermination(24, TimeUnit.HOURS); + if(!pool.awaitTermination(24, TimeUnit.HOURS)) { + log.warn("Timeout..."); + } log.info("All task finished."); log.info("Summary:"); @@ -150,6 +152,14 @@ public Integer call() { Globals.INSTANCE.stopAllSilent(); + var hasFailed = tasks.stream() + .filter(TaskWrapper.class::isInstance) + .anyMatch(t -> ((TaskWrapper) t).isFailed()); + + if (hasFailed) { + return Doer.EC_ERROR; + } + } catch (InterruptedException e) { log.error("oops", e); Thread.currentThread().interrupt(); diff --git a/src/main/java/io/github/s7i/doer/domain/Mappers.java b/src/main/java/io/github/s7i/doer/domain/Mappers.java index bb22716..8027fb4 100644 --- a/src/main/java/io/github/s7i/doer/domain/Mappers.java +++ b/src/main/java/io/github/s7i/doer/domain/Mappers.java @@ -4,23 +4,45 @@ import com.google.protobuf.BytesValue; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.StringValue; +import io.github.s7i.doer.DoerException; import io.github.s7i.doer.domain.output.Output; import io.github.s7i.doer.pipeline.proto.PipelineLoad; import io.github.s7i.doer.proto.Record; import lombok.experimental.UtilityClass; +import java.util.HashMap; + import static java.util.Objects.nonNull; @UtilityClass public class Mappers { public static Output.Load mapFrom(PipelineLoad value) throws InvalidProtocolBufferException { + if (!value.getLoad().is(Record.class)) { + throw new DoerException("invalid load type: " + value.getLoad().getTypeUrl()); + } var rec = value.getLoad().unpack(Record.class); - var load = Output.Load.builder() - .data(rec.getData().toByteArray()) - .build(); - return load; + var bld = Output.Load.builder(); + + if (rec.hasKey()) { + bld.key(rec.getKey().getValue()); + } + + if (rec.hasResource()) { + bld.resource(rec.getResource().getValue()); + } + + if (rec.hasData()) { + var recordData = rec.getData().getValue(); + bld.data(recordData.toByteArray()); + } + + if (!rec.getMetaMap().isEmpty()) { + bld.meta(new HashMap<>(rec.getMetaMap())); + } + + return bld.build(); } diff --git a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java index be65510..713bbec 100644 --- a/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java +++ b/src/main/java/io/github/s7i/doer/pipeline/grcp/GrpcInboundConnection.java @@ -1,7 +1,6 @@ package io.github.s7i.doer.pipeline.grcp; import com.google.protobuf.InvalidProtocolBufferException; -import io.github.s7i.doer.Doer; import io.github.s7i.doer.domain.Mappers; import io.github.s7i.doer.pipeline.BlockingPipe; import io.github.s7i.doer.pipeline.PipePuller; @@ -49,7 +48,7 @@ public void onNext(PipelineLoad value) { @Override public void onError(Throwable t) { - log.error("oops", t); + log.error("subscribe", t); } @Override @@ -61,7 +60,7 @@ public void onCompleted() { private void unload(PipelineLoad value) { try { - Doer.console().info("unpack {}", value); + log.debug("unpack {}", value); if (value.hasLoad()) { var load = Mappers.mapFrom(value); diff --git a/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy index f98cc0b..825eca1 100644 --- a/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy +++ b/src/test/groovy/io/github/s7i/doer/ingest/record/IngestRecordTest.groovy @@ -34,7 +34,6 @@ class IngestRecordTest extends Specification { expect: manifest.getKind() == 'ingest' records.size() == 3 - + (records[2] as Output.Load).getKey() == "key-record 3" } - } From 08ef5dab7858a083561c3788a8e91ea68cc3135d Mon Sep 17 00:00:00 2001 From: nefro85 Date: Mon, 27 Feb 2023 00:33:51 +0100 Subject: [PATCH 44/45] [REWORK] better tests missing file form last commit [TAG] Pipelines #14 --- .../github/s7i/doer/domain/MappersTest.groovy | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/test/groovy/io/github/s7i/doer/domain/MappersTest.groovy diff --git a/src/test/groovy/io/github/s7i/doer/domain/MappersTest.groovy b/src/test/groovy/io/github/s7i/doer/domain/MappersTest.groovy new file mode 100644 index 0000000..ff8e9a8 --- /dev/null +++ b/src/test/groovy/io/github/s7i/doer/domain/MappersTest.groovy @@ -0,0 +1,31 @@ +package io.github.s7i.doer.domain + +import com.google.protobuf.Any +import com.google.protobuf.ByteString +import com.google.protobuf.BytesValue +import com.google.protobuf.StringValue +import io.github.s7i.doer.pipeline.proto.PipelineLoad +import io.github.s7i.doer.proto.Record +import spock.lang.Specification + +class MappersTest extends Specification { + def "MapFrom PipelineLoad into Load"() { + + given: + def pipelineLoad = PipelineLoad.newBuilder() + .setLoad(Any.pack(Record.newBuilder() + .setData(BytesValue.of(ByteString.copyFromUtf8("this-is-my-string"))) + .setKey(StringValue.of("key-123")) + .setResource(StringValue.of("my-res")) + .putMeta("meta-key", "meta-value") + .build())) + .build() + when: + def load = Mappers.mapFrom(pipelineLoad) + then: + load.dataAsString() == "this-is-my-string" + load.getKey() == "key-123" + load.getResource() == "my-res" + load.getMeta().get("meta-key") == "meta-value" + } +} From 9f97f2ee3030341e9bfd16d015b196b3cd2d1356 Mon Sep 17 00:00:00 2001 From: nefro85 Date: Wed, 8 Nov 2023 02:45:00 +0100 Subject: [PATCH 45/45] =?UTF-8?q?[REWORK]=20=F0=9F=92=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/io/github/s7i/doer/command/util/Misc.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/github/s7i/doer/command/util/Misc.java b/src/main/java/io/github/s7i/doer/command/util/Misc.java index b89aa7c..3bad984 100644 --- a/src/main/java/io/github/s7i/doer/command/util/Misc.java +++ b/src/main/java/io/github/s7i/doer/command/util/Misc.java @@ -6,7 +6,6 @@ import io.github.s7i.doer.HandledRuntimeException; import io.github.s7i.doer.command.file.ReplaceInFile; import io.github.s7i.doer.domain.output.HttpOutput; -import io.github.s7i.doer.pipeline.PipelineService; import io.github.s7i.doer.util.Clipboard; import io.github.s7i.doer.util.GitProps; import io.github.s7i.doer.util.PropertyResolver; @@ -24,15 +23,12 @@ import org.mvel2.MVEL; import org.mvel2.compiler.CompiledExpression; import picocli.CommandLine; -import picocli.CommandLine.Parameters; -import picocli.CommandLine.Option; import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; import java.io.*; import java.net.URI; -import java.io.File; -import java.io.IOException; -import java.io.StringReader; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant;