Mixing
![convert text file with comma deliminator to csv [closed]](https://lh4.googleusercontent.com/-UUllNzz7XxY/AAAAAAAAAAI/AAAAAAAAACI/jpCmskrJeFM/s72-c/photo.jpg?sz=32)
Error spawning multiple threads to delete multiple same elements
I want to delete same elements replicated 2353218 times keeping only one, in the xml. Tried to spawn the process but getting following error. without spawn it is taking too much time. Please help.
xquery version "1.0-ml";
let $input := doc("http://www.somedomain.com/name/12345.xml")/xpath/toMultipleElement[2 to last()]
let $batch-size := 50000
let $input-size := fn:count($input)
let $num-batches := xs:int(math:ceil($input-size div $batch-size ))
let $result :=
<root>{
for $batch-start in (1 to $num-batches)
let $processing-seq := $input[($batch-size * ($batch-start - 1) + 1) to ($batch-size * ($batch-start ))]
return
xdmp:spawn-function(function() {
xdmp:node-delete($processing-seq),
<success batch-start='{$batch-start}'> processing sequence deleted</success>
},
<options xmlns="xdmp:eval">
<result>true</result>
<transaction-mode>update-auto-commit</transaction-mode>
</options>)
}</root>
return
xdmp:save("D:/batch-wise-delete.xml", $result)
Error: [1.0-ml] XDMP-DELEXTNODES: let $processing-seq := $input[$batch-size * ($batch-start - 1) + 1 to $batch-size * $batch-start] -- Cannot delete external nodes
marklogic marklogic-8
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
add a comment |
I want to delete same elements replicated 2353218 times keeping only one, in the xml. Tried to spawn the process but getting following error. without spawn it is taking too much time. Please help.
xquery version "1.0-ml";
let $input := doc("http://www.somedomain.com/name/12345.xml")/xpath/toMultipleElement[2 to last()]
let $batch-size := 50000
let $input-size := fn:count($input)
let $num-batches := xs:int(math:ceil($input-size div $batch-size ))
let $result :=
<root>{
for $batch-start in (1 to $num-batches)
let $processing-seq := $input[($batch-size * ($batch-start - 1) + 1) to ($batch-size * ($batch-start ))]
return
xdmp:spawn-function(function() {
xdmp:node-delete($processing-seq),
<success batch-start='{$batch-start}'> processing sequence deleted</success>
},
<options xmlns="xdmp:eval">
<result>true</result>
<transaction-mode>update-auto-commit</transaction-mode>
</options>)
}</root>
return
xdmp:save("D:/batch-wise-delete.xml", $result)
Error: [1.0-ml] XDMP-DELEXTNODES: let $processing-seq := $input[$batch-size * ($batch-start - 1) + 1 to $batch-size * $batch-start] -- Cannot delete external nodes
marklogic marklogic-8
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46
add a comment |
I want to delete same elements replicated 2353218 times keeping only one, in the xml. Tried to spawn the process but getting following error. without spawn it is taking too much time. Please help.
xquery version "1.0-ml";
let $input := doc("http://www.somedomain.com/name/12345.xml")/xpath/toMultipleElement[2 to last()]
let $batch-size := 50000
let $input-size := fn:count($input)
let $num-batches := xs:int(math:ceil($input-size div $batch-size ))
let $result :=
<root>{
for $batch-start in (1 to $num-batches)
let $processing-seq := $input[($batch-size * ($batch-start - 1) + 1) to ($batch-size * ($batch-start ))]
return
xdmp:spawn-function(function() {
xdmp:node-delete($processing-seq),
<success batch-start='{$batch-start}'> processing sequence deleted</success>
},
<options xmlns="xdmp:eval">
<result>true</result>
<transaction-mode>update-auto-commit</transaction-mode>
</options>)
}</root>
return
xdmp:save("D:/batch-wise-delete.xml", $result)
Error: [1.0-ml] XDMP-DELEXTNODES: let $processing-seq := $input[$batch-size * ($batch-start - 1) + 1 to $batch-size * $batch-start] -- Cannot delete external nodes
marklogic marklogic-8
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
I want to delete same elements replicated 2353218 times keeping only one, in the xml. Tried to spawn the process but getting following error. without spawn it is taking too much time. Please help.
xquery version "1.0-ml";
let $input := doc("http://www.somedomain.com/name/12345.xml")/xpath/toMultipleElement[2 to last()]
let $batch-size := 50000
let $input-size := fn:count($input)
let $num-batches := xs:int(math:ceil($input-size div $batch-size ))
let $result :=
<root>{
for $batch-start in (1 to $num-batches)
let $processing-seq := $input[($batch-size * ($batch-start - 1) + 1) to ($batch-size * ($batch-start ))]
return
xdmp:spawn-function(function() {
xdmp:node-delete($processing-seq),
<success batch-start='{$batch-start}'> processing sequence deleted</success>
},
<options xmlns="xdmp:eval">
<result>true</result>
<transaction-mode>update-auto-commit</transaction-mode>
</options>)
}</root>
return
xdmp:save("D:/batch-wise-delete.xml", $result)
Error: [1.0-ml] XDMP-DELEXTNODES: let $processing-seq := $input[$batch-size * ($batch-start - 1) + 1 to $batch-size * $batch-start] -- Cannot delete external nodes
marklogic marklogic-8
marklogic marklogic-8
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
asked Nov 19 '18 at 13:41
Shrikant Dhawale
356
356
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46
add a comment |
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46
add a comment |
2 Answers
2
active
oldest
votes
Instead of deleting all the children, just write a new parent having one child.
let $parent := doc("http://www.somedomain.com/name/12345.xml")/xpath/parent
let $chosen-child := $parent/toMultipleElement[1]
return xdmp:node-replace($parent, <parent>{ $chosen-child }</parent>
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
add a comment |
I recommend that instead of attempting to delete all the unwanted nodes that instead you reconstruct the document by inclusion in one pass.
The basic strategy is documented here https://developer.marklogic.com/blog/xquery-recursive-descent
essentially -- create a new document by recursing over all the nodes in the existing document and returning them unchanged except exlucde the unwanted nodes.
then save the new document over the old.
This can be done in one transaction very efficiently.
answered Nov 19 '18 at 15:01
DALDEI
3,15988
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53375893%2ferror-spawning-multiple-threads-to-delete-multiple-same-elements%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
2 Answers
2
active
oldest
votes
2 Answers
2
active
oldest
votes
active
oldest
votes
active
oldest
votes
Instead of deleting all the children, just write a new parent having one child.
let $parent := doc("http://www.somedomain.com/name/12345.xml")/xpath/parent
let $chosen-child := $parent/toMultipleElement[1]
return xdmp:node-replace($parent, <parent>{ $chosen-child }</parent>
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
add a comment |
Instead of deleting all the children, just write a new parent having one child.
let $parent := doc("http://www.somedomain.com/name/12345.xml")/xpath/parent
let $chosen-child := $parent/toMultipleElement[1]
return xdmp:node-replace($parent, <parent>{ $chosen-child }</parent>
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
add a comment |
Instead of deleting all the children, just write a new parent having one child.
let $parent := doc("http://www.somedomain.com/name/12345.xml")/xpath/parent
let $chosen-child := $parent/toMultipleElement[1]
return xdmp:node-replace($parent, <parent>{ $chosen-child }</parent>
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
Instead of deleting all the children, just write a new parent having one child.
let $parent := doc("http://www.somedomain.com/name/12345.xml")/xpath/parent
let $chosen-child := $parent/toMultipleElement[1]
return xdmp:node-replace($parent, <parent>{ $chosen-child }</parent>
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
answered Nov 21 '18 at 14:25
hunterhacker
1,79686
1,79686
add a comment |
add a comment |
I recommend that instead of attempting to delete all the unwanted nodes that instead you reconstruct the document by inclusion in one pass.
The basic strategy is documented here https://developer.marklogic.com/blog/xquery-recursive-descent
essentially -- create a new document by recursing over all the nodes in the existing document and returning them unchanged except exlucde the unwanted nodes.
then save the new document over the old.
This can be done in one transaction very efficiently.
answered Nov 19 '18 at 15:01
DALDEI
3,15988
add a comment |
I recommend that instead of attempting to delete all the unwanted nodes that instead you reconstruct the document by inclusion in one pass.
The basic strategy is documented here https://developer.marklogic.com/blog/xquery-recursive-descent
essentially -- create a new document by recursing over all the nodes in the existing document and returning them unchanged except exlucde the unwanted nodes.
then save the new document over the old.
This can be done in one transaction very efficiently.
answered Nov 19 '18 at 15:01
DALDEI
3,15988
add a comment |
I recommend that instead of attempting to delete all the unwanted nodes that instead you reconstruct the document by inclusion in one pass.
The basic strategy is documented here https://developer.marklogic.com/blog/xquery-recursive-descent
essentially -- create a new document by recursing over all the nodes in the existing document and returning them unchanged except exlucde the unwanted nodes.
then save the new document over the old.
This can be done in one transaction very efficiently.
answered Nov 19 '18 at 15:01
DALDEI
3,15988
I recommend that instead of attempting to delete all the unwanted nodes that instead you reconstruct the document by inclusion in one pass.
The basic strategy is documented here https://developer.marklogic.com/blog/xquery-recursive-descent
essentially -- create a new document by recursing over all the nodes in the existing document and returning them unchanged except exlucde the unwanted nodes.
then save the new document over the old.
This can be done in one transaction very efficiently.
answered Nov 19 '18 at 15:01
DALDEI
3,15988
answered Nov 19 '18 at 15:01
DALDEI
3,15988
answered Nov 19 '18 at 15:01
DALDEI
3,15988
answered Nov 19 '18 at 15:01
DALDEI
3,15988
3,15988
add a comment |
add a comment |
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Some of your past answers have not been well-received, and you're in danger of being blocked from answering.
Please pay close attention to the following guidance:
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53375893%2ferror-spawning-multiple-threads-to-delete-multiple-same-elements%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
I have the impression you are trying to omit elements from a long list, and save the filtered list to disk. You wouldn't use node-delete for that, but rather reconstruct the doc while omitting what you don't need. Can you confirm?
– grtjn
Nov 19 '18 at 16:25
@grtjn, Actually one of the element in xml file got duplicated millions of time so I want to keep only first and delete rest all, hence I can restore original doc as it is. the operation report I m storing to disk, i.e how many time delete got called.
– Shrikant Dhawale
Nov 19 '18 at 16:30
The message 'cannot delete external nodes' is related to the spawning. You can pass nodes through, but you'll get a copy, which you can no longer node-update. You'd have to pass through doc-uri, and position info to retrieve a fresh copy of the node for deleting. But this is not an efficient way to get rid of so many elements in one doc..
– grtjn
Nov 19 '18 at 16:36
so is there any efficient work around for this
– Shrikant Dhawale
Nov 19 '18 at 16:44
Yes, see the answer by @DALDEI..
– grtjn
Nov 19 '18 at 18:46