Mixing
array of promises containing cassandra queris not triggering well
up vote
1
down vote
favorite
I'm currently trying to fill a cassandra table with the content of an xlsx file, but i'm facing a problem.
I managed to do an array with all the queries necessary to fill my table ("insert into my_table (values name, ...) values (values, ...);" ).
So my array got like 7000 strings.
I'm then doing a promise and inside this promise I made a loop to fill and array of promises in order to trigger the first promise when all the promises are over.
this is the code I made
index.js =>
const ImportFileContent = require("./scripts/import_file_content")
const InsertDb = require("./scripts/insertDb")
const cassandra = require('cassandra-driver');
const databaseConfig = require('./config/database.json');
const authProvider = new cassandra.auth.PlainTextAuthProvider(databaseConfig.cassandra.username, databaseConfig.cassandra.password);
const db = new cassandra.Client({
contactPoints: databaseConfig.cassandra.contactPoints,
authProvider: authProvider
});
// ImportFileContent.importFileContent return an array of string, those strings contains all the 7000+ queries
ImportFileContent.importFileContent().then(queries => {
InsertDb.clients(db, queries).then(result => {
console.log(result);
db.shutdown(function (err, result) {
});
});
});
insertDb.js =>
let DB = null;
module.exports = {
ClientsLeasing: function (db, queries) {
DB = db;
return insertClientsLeasing(queries);
}
}
function insertClientsLeasing(queries) {
return new Promise((resolve, reject) => {
let nbError = 0;
let nbSuccess = 0;
let promisesArray = ;
//I made i <2000 here because my cassandra setup doesn't manage more than 2048 request in parallele
for (let i = 0; i < 2000; i++) {
promisesArray.push(new Promise(function (resolve, reject) {
DB.execute(queries[i], function (err, result) {
if (err) {
nbError++;
reject(err)
} else {
nbSuccess++;
resolve();
}
});
}));
}
Promise.all(promisesArray).then((result) => {
console.log("is over")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("success");
}).catch((error) => {
console.log(error);
console.log("is over error")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("error");
});
});
}
My table got two primary keys the creation date (wich is now()) and the client id that may be in multiple rows of the xlsx (that can be the source of the problem ?).
So now when i launch this code my output is 
then when i do a count in the table via cqls i only get 1962 rows with this output
I feel like i'm missing something in the way I use my promises. I don't really get it.
Thanks
javascript node.js cassandra es6-promise
asked Nov 19 at 9:18
lanetrotro
669
add a comment |
up vote
1
down vote
favorite
I'm currently trying to fill a cassandra table with the content of an xlsx file, but i'm facing a problem.
I managed to do an array with all the queries necessary to fill my table ("insert into my_table (values name, ...) values (values, ...);" ).
So my array got like 7000 strings.
I'm then doing a promise and inside this promise I made a loop to fill and array of promises in order to trigger the first promise when all the promises are over.
this is the code I made
index.js =>
const ImportFileContent = require("./scripts/import_file_content")
const InsertDb = require("./scripts/insertDb")
const cassandra = require('cassandra-driver');
const databaseConfig = require('./config/database.json');
const authProvider = new cassandra.auth.PlainTextAuthProvider(databaseConfig.cassandra.username, databaseConfig.cassandra.password);
const db = new cassandra.Client({
contactPoints: databaseConfig.cassandra.contactPoints,
authProvider: authProvider
});
// ImportFileContent.importFileContent return an array of string, those strings contains all the 7000+ queries
ImportFileContent.importFileContent().then(queries => {
InsertDb.clients(db, queries).then(result => {
console.log(result);
db.shutdown(function (err, result) {
});
});
});
insertDb.js =>
let DB = null;
module.exports = {
ClientsLeasing: function (db, queries) {
DB = db;
return insertClientsLeasing(queries);
}
}
function insertClientsLeasing(queries) {
return new Promise((resolve, reject) => {
let nbError = 0;
let nbSuccess = 0;
let promisesArray = ;
//I made i <2000 here because my cassandra setup doesn't manage more than 2048 request in parallele
for (let i = 0; i < 2000; i++) {
promisesArray.push(new Promise(function (resolve, reject) {
DB.execute(queries[i], function (err, result) {
if (err) {
nbError++;
reject(err)
} else {
nbSuccess++;
resolve();
}
});
}));
}
Promise.all(promisesArray).then((result) => {
console.log("is over")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("success");
}).catch((error) => {
console.log(error);
console.log("is over error")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("error");
});
});
}
My table got two primary keys the creation date (wich is now()) and the client id that may be in multiple rows of the xlsx (that can be the source of the problem ?).
So now when i launch this code my output is
then when i do a count in the table via cqls i only get 1962 rows with this output
I feel like i'm missing something in the way I use my promises. I don't really get it.
Thanks
javascript node.js cassandra es6-promise
asked Nov 19 at 9:18
lanetrotro
669
2
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago
add a comment |
up vote
1
down vote
favorite
up vote
1
down vote
favorite
I'm currently trying to fill a cassandra table with the content of an xlsx file, but i'm facing a problem.
I managed to do an array with all the queries necessary to fill my table ("insert into my_table (values name, ...) values (values, ...);" ).
So my array got like 7000 strings.
I'm then doing a promise and inside this promise I made a loop to fill and array of promises in order to trigger the first promise when all the promises are over.
this is the code I made
index.js =>
const ImportFileContent = require("./scripts/import_file_content")
const InsertDb = require("./scripts/insertDb")
const cassandra = require('cassandra-driver');
const databaseConfig = require('./config/database.json');
const authProvider = new cassandra.auth.PlainTextAuthProvider(databaseConfig.cassandra.username, databaseConfig.cassandra.password);
const db = new cassandra.Client({
contactPoints: databaseConfig.cassandra.contactPoints,
authProvider: authProvider
});
// ImportFileContent.importFileContent return an array of string, those strings contains all the 7000+ queries
ImportFileContent.importFileContent().then(queries => {
InsertDb.clients(db, queries).then(result => {
console.log(result);
db.shutdown(function (err, result) {
});
});
});
insertDb.js =>
let DB = null;
module.exports = {
ClientsLeasing: function (db, queries) {
DB = db;
return insertClientsLeasing(queries);
}
}
function insertClientsLeasing(queries) {
return new Promise((resolve, reject) => {
let nbError = 0;
let nbSuccess = 0;
let promisesArray = ;
//I made i <2000 here because my cassandra setup doesn't manage more than 2048 request in parallele
for (let i = 0; i < 2000; i++) {
promisesArray.push(new Promise(function (resolve, reject) {
DB.execute(queries[i], function (err, result) {
if (err) {
nbError++;
reject(err)
} else {
nbSuccess++;
resolve();
}
});
}));
}
Promise.all(promisesArray).then((result) => {
console.log("is over")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("success");
}).catch((error) => {
console.log(error);
console.log("is over error")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("error");
});
});
}
My table got two primary keys the creation date (wich is now()) and the client id that may be in multiple rows of the xlsx (that can be the source of the problem ?).
So now when i launch this code my output is
then when i do a count in the table via cqls i only get 1962 rows with this output
I feel like i'm missing something in the way I use my promises. I don't really get it.
Thanks
javascript node.js cassandra es6-promise
asked Nov 19 at 9:18
lanetrotro
669
I'm currently trying to fill a cassandra table with the content of an xlsx file, but i'm facing a problem.
I managed to do an array with all the queries necessary to fill my table ("insert into my_table (values name, ...) values (values, ...);" ).
So my array got like 7000 strings.
I'm then doing a promise and inside this promise I made a loop to fill and array of promises in order to trigger the first promise when all the promises are over.
this is the code I made
index.js =>
const ImportFileContent = require("./scripts/import_file_content")
const InsertDb = require("./scripts/insertDb")
const cassandra = require('cassandra-driver');
const databaseConfig = require('./config/database.json');
const authProvider = new cassandra.auth.PlainTextAuthProvider(databaseConfig.cassandra.username, databaseConfig.cassandra.password);
const db = new cassandra.Client({
contactPoints: databaseConfig.cassandra.contactPoints,
authProvider: authProvider
});
// ImportFileContent.importFileContent return an array of string, those strings contains all the 7000+ queries
ImportFileContent.importFileContent().then(queries => {
InsertDb.clients(db, queries).then(result => {
console.log(result);
db.shutdown(function (err, result) {
});
});
});
insertDb.js =>
let DB = null;
module.exports = {
ClientsLeasing: function (db, queries) {
DB = db;
return insertClientsLeasing(queries);
}
}
function insertClientsLeasing(queries) {
return new Promise((resolve, reject) => {
let nbError = 0;
let nbSuccess = 0;
let promisesArray = ;
//I made i <2000 here because my cassandra setup doesn't manage more than 2048 request in parallele
for (let i = 0; i < 2000; i++) {
promisesArray.push(new Promise(function (resolve, reject) {
DB.execute(queries[i], function (err, result) {
if (err) {
nbError++;
reject(err)
} else {
nbSuccess++;
resolve();
}
});
}));
}
Promise.all(promisesArray).then((result) => {
console.log("is over")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("success");
}).catch((error) => {
console.log(error);
console.log("is over error")
console.log("over ===================== success => ", nbSuccess, " errors => ", nbError);
resolve("error");
});
});
}
My table got two primary keys the creation date (wich is now()) and the client id that may be in multiple rows of the xlsx (that can be the source of the problem ?).
So now when i launch this code my output is
then when i do a count in the table via cqls i only get 1962 rows with this output
I feel like i'm missing something in the way I use my promises. I don't really get it.
Thanks
javascript node.js cassandra es6-promise
javascript node.js cassandra es6-promise
asked Nov 19 at 9:18
lanetrotro
669
asked Nov 19 at 9:18
lanetrotro
669
edited 2 days ago
asked Nov 19 at 9:18
lanetrotro
669
asked Nov 19 at 9:18
lanetrotro
669
asked Nov 19 at 9:18
lanetrotro
669
669
2
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago
add a comment |
2
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago
2
2
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago
add a comment |
active
oldest
votes
active
oldest
votes
active
oldest
votes
active
oldest
votes
active
oldest
votes
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53371507%2farray-of-promises-containing-cassandra-queris-not-triggering-well%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
2
please add the table schema, and queries that you execute - it could be that your primary key definition isn't correct, and you simply overwrite the data...
– Alex Ott
2 days ago
the table i'm trying to insert the value has two primary keys, the date of insert (now()) and a client id. that client id may be in multiple rows, is that the source of the problem ? I thought now was used to create uuid ? here it's just for the date but it's used in the primary key. I'm working with an existing table and i can't really change it
– lanetrotro
2 days ago
ok so you were right the problem was that i overwrited some data. thanks for the help :)
– lanetrotro
2 days ago