Mixing
Can Master step act like a Slave after sending partitioning messages while using...
.everyoneloves__top-leaderboard:empty,.everyoneloves__mid-leaderboard:empty,.everyoneloves__bot-mid-leaderboard:empty{ height:90px;width:728px;box-sizing:border-box;
}
1
I have created a small Proof of Concept(POC) using Spring Batch that uses MessageChannelPartitionHandler. The idea is to deploy the code on multiple nodes in production. My question is can the master node do the job of a slave also besides sending the partitioning messages to the slaves using messaging middleware like ActiveMQ. Is the master restricted to only sending the messages?
@Bean
public Step masterStep() throws Exception {
return stepBuilderFactory.get("masterStep")
.partitioner(slaveStep().getName(), partitioner())
.step(slaveStep())
.partitionHandler(partitionHandler(null))
.taskExecutor(taskExecutor())
.gridSize(GRID_SIZE)
.build();
}
@Bean
public Step slaveStep() {
return stepBuilderFactory.get("slaveStep")
.<WorkItem, ReportData>chunk(CHUNK_SIZE)
.reader(reader())
.processor(processor())
.writer(writer())
.build();
}
@Bean
@Profile("master")
public Job processingBatchJob() throws Exception {
return
jobBuilderFactory.get("processingBatchJob").listener(jobListener)
.start(masterStep())
.build();
}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(20);
taskExecutor.setCorePoolSize(20);
taskExecutor.setQueueCapacity(100);
taskExecutor.afterPropertiesSet();
taskExecutor.setThreadNamePrefix("xtrac");
return taskExecutor;
}
@Bean
public PartitionHandler partitionHandler(MessagingTemplate
messagingTemplate) throws Exception
{
MessageChannelPartitionHandler partitionHandler = new
MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setMessagingOperations(this.messageTemplate);
partitionHandler.setPollInterval(5000l);
partitionHandler.setJobExplorer(this.jobExplorer);
partitionHandler.afterPropertiesSet();
return partitionHandler;
}
spring-batch spring-jms
asked Jan 3 at 4:04
HinfyHinfy
62
add a comment |
1
I have created a small Proof of Concept(POC) using Spring Batch that uses MessageChannelPartitionHandler. The idea is to deploy the code on multiple nodes in production. My question is can the master node do the job of a slave also besides sending the partitioning messages to the slaves using messaging middleware like ActiveMQ. Is the master restricted to only sending the messages?
@Bean
public Step masterStep() throws Exception {
return stepBuilderFactory.get("masterStep")
.partitioner(slaveStep().getName(), partitioner())
.step(slaveStep())
.partitionHandler(partitionHandler(null))
.taskExecutor(taskExecutor())
.gridSize(GRID_SIZE)
.build();
}
@Bean
public Step slaveStep() {
return stepBuilderFactory.get("slaveStep")
.<WorkItem, ReportData>chunk(CHUNK_SIZE)
.reader(reader())
.processor(processor())
.writer(writer())
.build();
}
@Bean
@Profile("master")
public Job processingBatchJob() throws Exception {
return
jobBuilderFactory.get("processingBatchJob").listener(jobListener)
.start(masterStep())
.build();
}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(20);
taskExecutor.setCorePoolSize(20);
taskExecutor.setQueueCapacity(100);
taskExecutor.afterPropertiesSet();
taskExecutor.setThreadNamePrefix("xtrac");
return taskExecutor;
}
@Bean
public PartitionHandler partitionHandler(MessagingTemplate
messagingTemplate) throws Exception
{
MessageChannelPartitionHandler partitionHandler = new
MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setMessagingOperations(this.messageTemplate);
partitionHandler.setPollInterval(5000l);
partitionHandler.setJobExplorer(this.jobExplorer);
partitionHandler.afterPropertiesSet();
return partitionHandler;
}
spring-batch spring-jms
asked Jan 3 at 4:04
HinfyHinfy
62
add a comment |
1
1
1
1
I have created a small Proof of Concept(POC) using Spring Batch that uses MessageChannelPartitionHandler. The idea is to deploy the code on multiple nodes in production. My question is can the master node do the job of a slave also besides sending the partitioning messages to the slaves using messaging middleware like ActiveMQ. Is the master restricted to only sending the messages?
@Bean
public Step masterStep() throws Exception {
return stepBuilderFactory.get("masterStep")
.partitioner(slaveStep().getName(), partitioner())
.step(slaveStep())
.partitionHandler(partitionHandler(null))
.taskExecutor(taskExecutor())
.gridSize(GRID_SIZE)
.build();
}
@Bean
public Step slaveStep() {
return stepBuilderFactory.get("slaveStep")
.<WorkItem, ReportData>chunk(CHUNK_SIZE)
.reader(reader())
.processor(processor())
.writer(writer())
.build();
}
@Bean
@Profile("master")
public Job processingBatchJob() throws Exception {
return
jobBuilderFactory.get("processingBatchJob").listener(jobListener)
.start(masterStep())
.build();
}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(20);
taskExecutor.setCorePoolSize(20);
taskExecutor.setQueueCapacity(100);
taskExecutor.afterPropertiesSet();
taskExecutor.setThreadNamePrefix("xtrac");
return taskExecutor;
}
@Bean
public PartitionHandler partitionHandler(MessagingTemplate
messagingTemplate) throws Exception
{
MessageChannelPartitionHandler partitionHandler = new
MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setMessagingOperations(this.messageTemplate);
partitionHandler.setPollInterval(5000l);
partitionHandler.setJobExplorer(this.jobExplorer);
partitionHandler.afterPropertiesSet();
return partitionHandler;
}
spring-batch spring-jms
asked Jan 3 at 4:04
HinfyHinfy
62
I have created a small Proof of Concept(POC) using Spring Batch that uses MessageChannelPartitionHandler. The idea is to deploy the code on multiple nodes in production. My question is can the master node do the job of a slave also besides sending the partitioning messages to the slaves using messaging middleware like ActiveMQ. Is the master restricted to only sending the messages?
@Bean
public Step masterStep() throws Exception {
return stepBuilderFactory.get("masterStep")
.partitioner(slaveStep().getName(), partitioner())
.step(slaveStep())
.partitionHandler(partitionHandler(null))
.taskExecutor(taskExecutor())
.gridSize(GRID_SIZE)
.build();
}
@Bean
public Step slaveStep() {
return stepBuilderFactory.get("slaveStep")
.<WorkItem, ReportData>chunk(CHUNK_SIZE)
.reader(reader())
.processor(processor())
.writer(writer())
.build();
}
@Bean
@Profile("master")
public Job processingBatchJob() throws Exception {
return
jobBuilderFactory.get("processingBatchJob").listener(jobListener)
.start(masterStep())
.build();
}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(20);
taskExecutor.setCorePoolSize(20);
taskExecutor.setQueueCapacity(100);
taskExecutor.afterPropertiesSet();
taskExecutor.setThreadNamePrefix("xtrac");
return taskExecutor;
}
@Bean
public PartitionHandler partitionHandler(MessagingTemplate
messagingTemplate) throws Exception
{
MessageChannelPartitionHandler partitionHandler = new
MessageChannelPartitionHandler();
partitionHandler.setStepName("slaveStep");
partitionHandler.setGridSize(GRID_SIZE);
partitionHandler.setMessagingOperations(this.messageTemplate);
partitionHandler.setPollInterval(5000l);
partitionHandler.setJobExplorer(this.jobExplorer);
partitionHandler.afterPropertiesSet();
return partitionHandler;
}
spring-batch spring-jms
spring-batch spring-jms
asked Jan 3 at 4:04
HinfyHinfy
62
asked Jan 3 at 4:04
HinfyHinfy
62
asked Jan 3 at 4:04
HinfyHinfy
62
asked Jan 3 at 4:04
HinfyHinfy
62
asked Jan 3 at 4:04
HinfyHinfy
62
62
add a comment |
add a comment |
1 Answer
1
active
oldest
votes
0
Yes, the master can act as a worker in a remote partitioning setup and is not restricted to only sending messages to and aggregating results back from workers. I would even recommend to do it so that resources on the master node are not wasted waiting for workers to finish their work.
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54016203%2fcan-master-step-act-like-a-slave-after-sending-partitioning-messages-while-using%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
1 Answer
1
active
oldest
votes
1 Answer
1
active
oldest
votes
active
oldest
votes
active
oldest
votes
0
Yes, the master can act as a worker in a remote partitioning setup and is not restricted to only sending messages to and aggregating results back from workers. I would even recommend to do it so that resources on the master node are not wasted waiting for workers to finish their work.
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
add a comment |
0
Yes, the master can act as a worker in a remote partitioning setup and is not restricted to only sending messages to and aggregating results back from workers. I would even recommend to do it so that resources on the master node are not wasted waiting for workers to finish their work.
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
add a comment |
0
0
0
Yes, the master can act as a worker in a remote partitioning setup and is not restricted to only sending messages to and aggregating results back from workers. I would even recommend to do it so that resources on the master node are not wasted waiting for workers to finish their work.
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
Yes, the master can act as a worker in a remote partitioning setup and is not restricted to only sending messages to and aggregating results back from workers. I would even recommend to do it so that resources on the master node are not wasted waiting for workers to finish their work.
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
answered Jan 3 at 8:43
Mahmoud Ben HassineMahmoud Ben Hassine
5,5701717
5,5701717
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
add a comment |
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
Can you share code snippets please. I could not locate on Spring site or on SO.
– Hinfy
Jan 3 at 10:36
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
I don't have a ready to use example but you can make the master act as a worker by activating the worker's code in the master's application context (add the listener on the request queue). Another option is to use Spring Profiles (see this repo for examples) and launch the master with both the "master" and "worker" profiles.
– Mahmoud Ben Hassine
Jan 3 at 14:42
add a comment |
draft saved
draft discarded
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
draft saved
draft discarded
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f54016203%2fcan-master-step-act-like-a-slave-after-sending-partitioning-messages-while-using%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
