001/* 002 * Copyright 2016-2019 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2016-2019 Ping Identity Corporation 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldap.sdk.transformations; 022 023 024 025import java.io.File; 026import java.io.FileOutputStream; 027import java.io.InputStream; 028import java.io.OutputStream; 029import java.util.ArrayList; 030import java.util.EnumSet; 031import java.util.Iterator; 032import java.util.LinkedHashMap; 033import java.util.List; 034import java.util.Set; 035import java.util.TreeMap; 036import java.util.concurrent.atomic.AtomicLong; 037import java.util.zip.GZIPOutputStream; 038 039import com.unboundid.ldap.sdk.Attribute; 040import com.unboundid.ldap.sdk.ChangeType; 041import com.unboundid.ldap.sdk.DN; 042import com.unboundid.ldap.sdk.Entry; 043import com.unboundid.ldap.sdk.LDAPException; 044import com.unboundid.ldap.sdk.ResultCode; 045import com.unboundid.ldap.sdk.Version; 046import com.unboundid.ldap.sdk.schema.Schema; 047import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 048import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 049import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 050import com.unboundid.ldif.LDIFException; 051import com.unboundid.ldif.LDIFReader; 052import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 053import com.unboundid.ldif.LDIFReaderEntryTranslator; 054import com.unboundid.ldif.LDIFRecord; 055import com.unboundid.util.ByteStringBuffer; 056import com.unboundid.util.CommandLineTool; 057import com.unboundid.util.Debug; 058import com.unboundid.util.ObjectPair; 059import com.unboundid.util.PassphraseEncryptedOutputStream; 060import com.unboundid.util.StaticUtils; 061import com.unboundid.util.ThreadSafety; 062import com.unboundid.util.ThreadSafetyLevel; 063import com.unboundid.util.args.ArgumentException; 064import com.unboundid.util.args.ArgumentParser; 065import com.unboundid.util.args.BooleanArgument; 066import com.unboundid.util.args.DNArgument; 067import com.unboundid.util.args.FileArgument; 068import com.unboundid.util.args.FilterArgument; 069import com.unboundid.util.args.IntegerArgument; 070import com.unboundid.util.args.ScopeArgument; 071import com.unboundid.util.args.StringArgument; 072 073import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 074 075 076 077/** 078 * This class provides a command-line tool that can be used to apply a number of 079 * transformations to an LDIF file. The transformations that can be applied 080 * include: 081 * <UL> 082 * <LI> 083 * It can scramble the values of a specified set of attributes in a manner 084 * that attempts to preserve the syntax and consistently scrambles the same 085 * value to the same representation. 086 * </LI> 087 * <LI> 088 * It can strip a specified set of attributes out of entries. 089 * </LI> 090 * <LI> 091 * It can redact the values of a specified set of attributes, to indicate 092 * that the values are there but providing no information about what their 093 * values are. 094 * </LI> 095 * <LI> 096 * It can replace the values of a specified attribute with a given set of 097 * values. 098 * </LI> 099 * <LI> 100 * It can add an attribute with a given set of values to any entry that does 101 * not contain that attribute. 102 * </LI> 103 * <LI> 104 * It can replace the values of a specified attribute with a value that 105 * contains a sequentially-incrementing counter. 106 * </LI> 107 * <LI> 108 * It can strip entries matching a given base DN, scope, and filter out of 109 * the LDIF file. 110 * </LI> 111 * <LI> 112 * It can perform DN mapping, so that entries that exist below one base DN 113 * are moved below a different base DN. 114 * </LI> 115 * <LI> 116 * It can perform attribute mapping, to replace uses of one attribute name 117 * with another. 118 * </LI> 119 * </UL> 120 */ 121@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 122public final class TransformLDIF 123 extends CommandLineTool 124 implements LDIFReaderEntryTranslator 125{ 126 /** 127 * The maximum length of any message to write to standard output or standard 128 * error. 129 */ 130 private static final int MAX_OUTPUT_LINE_LENGTH = 131 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 132 133 134 135 // The arguments for use by this program. 136 private BooleanArgument addToExistingValues = null; 137 private BooleanArgument appendToTargetLDIF = null; 138 private BooleanArgument compressTarget = null; 139 private BooleanArgument encryptTarget = null; 140 private BooleanArgument excludeRecordsWithoutChangeType = null; 141 private BooleanArgument excludeNonMatchingEntries = null; 142 private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null; 143 private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 144 private BooleanArgument hideRedactedValueCount = null; 145 private BooleanArgument processDNs = null; 146 private BooleanArgument sourceCompressed = null; 147 private BooleanArgument sourceContainsChangeRecords = null; 148 private BooleanArgument sourceFromStandardInput = null; 149 private BooleanArgument targetToStandardOutput = null; 150 private DNArgument addAttributeBaseDN = null; 151 private DNArgument excludeEntryBaseDN = null; 152 private DNArgument flattenBaseDN = null; 153 private DNArgument moveSubtreeFrom = null; 154 private DNArgument moveSubtreeTo = null; 155 private FileArgument encryptionPassphraseFile = null; 156 private FileArgument schemaPath = null; 157 private FileArgument sourceLDIF = null; 158 private FileArgument targetLDIF = null; 159 private FilterArgument addAttributeFilter = null; 160 private FilterArgument excludeEntryFilter = null; 161 private FilterArgument flattenExcludeFilter = null; 162 private IntegerArgument initialSequentialValue = null; 163 private IntegerArgument numThreads = null; 164 private IntegerArgument randomSeed = null; 165 private IntegerArgument sequentialValueIncrement = null; 166 private IntegerArgument wrapColumn = null; 167 private ScopeArgument addAttributeScope = null; 168 private ScopeArgument excludeEntryScope = null; 169 private StringArgument addAttributeName = null; 170 private StringArgument addAttributeValue = null; 171 private StringArgument excludeAttribute = null; 172 private StringArgument excludeChangeType = null; 173 private StringArgument redactAttribute = null; 174 private StringArgument renameAttributeFrom = null; 175 private StringArgument renameAttributeTo = null; 176 private StringArgument replaceValuesAttribute = null; 177 private StringArgument replacementValue = null; 178 private StringArgument scrambleAttribute = null; 179 private StringArgument scrambleJSONField = null; 180 private StringArgument sequentialAttribute = null; 181 private StringArgument textAfterSequentialValue = null; 182 private StringArgument textBeforeSequentialValue = null; 183 184 // A set of thread-local byte stream buffers that will be used to construct 185 // the LDIF representations of records. 186 private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 187 new ThreadLocal<>(); 188 189 190 191 /** 192 * Invokes this tool with the provided set of arguments. 193 * 194 * @param args The command-line arguments provided to this program. 195 */ 196 public static void main(final String... args) 197 { 198 final ResultCode resultCode = main(System.out, System.err, args); 199 if (resultCode != ResultCode.SUCCESS) 200 { 201 System.exit(resultCode.intValue()); 202 } 203 } 204 205 206 207 /** 208 * Invokes this tool with the provided set of arguments. 209 * 210 * @param out The output stream to use for standard output. It may be 211 * {@code null} if standard output should be suppressed. 212 * @param err The output stream to use for standard error. It may be 213 * {@code null} if standard error should be suppressed. 214 * @param args The command-line arguments provided to this program. 215 * 216 * @return A result code indicating whether processing completed 217 * successfully. 218 */ 219 public static ResultCode main(final OutputStream out, final OutputStream err, 220 final String... args) 221 { 222 final TransformLDIF tool = new TransformLDIF(out, err); 223 return tool.runTool(args); 224 } 225 226 227 228 /** 229 * Creates a new instance of this tool with the provided information. 230 * 231 * @param out The output stream to use for standard output. It may be 232 * {@code null} if standard output should be suppressed. 233 * @param err The output stream to use for standard error. It may be 234 * {@code null} if standard error should be suppressed. 235 */ 236 public TransformLDIF(final OutputStream out, final OutputStream err) 237 { 238 super(out, err); 239 } 240 241 242 243 /** 244 * {@inheritDoc} 245 */ 246 @Override() 247 public String getToolName() 248 { 249 return "transform-ldif"; 250 } 251 252 253 254 /** 255 * {@inheritDoc} 256 */ 257 @Override() 258 public String getToolDescription() 259 { 260 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 261 } 262 263 264 265 /** 266 * {@inheritDoc} 267 */ 268 @Override() 269 public String getToolVersion() 270 { 271 return Version.NUMERIC_VERSION_STRING; 272 } 273 274 275 276 /** 277 * {@inheritDoc} 278 */ 279 @Override() 280 public boolean supportsInteractiveMode() 281 { 282 return true; 283 } 284 285 286 287 /** 288 * {@inheritDoc} 289 */ 290 @Override() 291 public boolean defaultsToInteractiveMode() 292 { 293 return true; 294 } 295 296 297 298 /** 299 * {@inheritDoc} 300 */ 301 @Override() 302 public boolean supportsPropertiesFile() 303 { 304 return true; 305 } 306 307 308 309 /** 310 * {@inheritDoc} 311 */ 312 @Override() 313 public void addToolArguments(final ArgumentParser parser) 314 throws ArgumentException 315 { 316 // Add arguments pertaining to the source and target LDIF files. 317 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 318 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 319 false); 320 sourceLDIF.addLongIdentifier("inputLDIF", true); 321 sourceLDIF.addLongIdentifier("source-ldif", true); 322 sourceLDIF.addLongIdentifier("input-ldif", true); 323 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 324 parser.addArgument(sourceLDIF); 325 326 sourceFromStandardInput = new BooleanArgument(null, 327 "sourceFromStandardInput", 1, 328 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 329 sourceFromStandardInput.addLongIdentifier("source-from-standard-input", 330 true); 331 sourceFromStandardInput.setArgumentGroupName( 332 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 333 parser.addArgument(sourceFromStandardInput); 334 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 335 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 336 337 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 338 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 339 false); 340 targetLDIF.addLongIdentifier("outputLDIF", true); 341 targetLDIF.addLongIdentifier("target-ldif", true); 342 targetLDIF.addLongIdentifier("output-ldif", true); 343 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 344 parser.addArgument(targetLDIF); 345 346 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 347 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 348 targetToStandardOutput.addLongIdentifier("target-to-standard-output", true); 349 targetToStandardOutput.setArgumentGroupName( 350 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 351 parser.addArgument(targetToStandardOutput); 352 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 353 354 sourceContainsChangeRecords = new BooleanArgument(null, 355 "sourceContainsChangeRecords", 356 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 357 sourceContainsChangeRecords.addLongIdentifier( 358 "source-contains-change-records", true); 359 sourceContainsChangeRecords.setArgumentGroupName( 360 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 361 parser.addArgument(sourceContainsChangeRecords); 362 363 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 364 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 365 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true); 366 appendToTargetLDIF.setArgumentGroupName( 367 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 368 parser.addArgument(appendToTargetLDIF); 369 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 370 371 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 372 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 373 wrapColumn.addLongIdentifier("wrap-column", true); 374 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 375 parser.addArgument(wrapColumn); 376 377 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 378 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 379 sourceCompressed.addLongIdentifier("inputCompressed", true); 380 sourceCompressed.addLongIdentifier("source-compressed", true); 381 sourceCompressed.addLongIdentifier("input-compressed", true); 382 sourceCompressed.setArgumentGroupName( 383 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 384 parser.addArgument(sourceCompressed); 385 386 compressTarget = new BooleanArgument('c', "compressTarget", 387 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 388 compressTarget.addLongIdentifier("compressOutput", true); 389 compressTarget.addLongIdentifier("compress", true); 390 compressTarget.addLongIdentifier("compress-target", true); 391 compressTarget.addLongIdentifier("compress-output", true); 392 compressTarget.setArgumentGroupName( 393 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 394 parser.addArgument(compressTarget); 395 396 encryptTarget = new BooleanArgument(null, "encryptTarget", 397 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get()); 398 encryptTarget.addLongIdentifier("encryptOutput", true); 399 encryptTarget.addLongIdentifier("encrypt", true); 400 encryptTarget.addLongIdentifier("encrypt-target", true); 401 encryptTarget.addLongIdentifier("encrypt-output", true); 402 encryptTarget.setArgumentGroupName( 403 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 404 parser.addArgument(encryptTarget); 405 406 encryptionPassphraseFile = new FileArgument(null, 407 "encryptionPassphraseFile", false, 1, null, 408 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true, 409 true, false); 410 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 411 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 412 true); 413 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 414 true); 415 encryptionPassphraseFile.setArgumentGroupName( 416 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 417 parser.addArgument(encryptionPassphraseFile); 418 419 420 // Add arguments pertaining to attribute scrambling. 421 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 422 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 423 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 424 scrambleAttribute.addLongIdentifier("attributeName", true); 425 scrambleAttribute.addLongIdentifier("scramble-attribute", true); 426 scrambleAttribute.addLongIdentifier("attribute-name", true); 427 scrambleAttribute.setArgumentGroupName( 428 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 429 parser.addArgument(scrambleAttribute); 430 431 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 432 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 433 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 434 scrambleAttribute.getIdentifierString())); 435 scrambleJSONField.addLongIdentifier("scramble-json-field", true); 436 scrambleJSONField.setArgumentGroupName( 437 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 438 parser.addArgument(scrambleJSONField); 439 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 440 441 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 442 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 443 randomSeed.addLongIdentifier("random-seed", true); 444 randomSeed.setArgumentGroupName( 445 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 446 parser.addArgument(randomSeed); 447 448 449 // Add arguments pertaining to replacing attribute values with a generated 450 // value using a sequential counter. 451 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 452 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 453 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 454 sourceContainsChangeRecords.getIdentifierString())); 455 sequentialAttribute.addLongIdentifier("sequentialAttributeName", true); 456 sequentialAttribute.addLongIdentifier("sequential-attribute", true); 457 sequentialAttribute.addLongIdentifier("sequential-attribute-name", true); 458 sequentialAttribute.setArgumentGroupName( 459 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 460 parser.addArgument(sequentialAttribute); 461 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 462 sequentialAttribute); 463 464 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 465 false, 1, null, 466 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 467 sequentialAttribute.getIdentifierString())); 468 initialSequentialValue.addLongIdentifier("initial-sequential-value", true); 469 initialSequentialValue.setArgumentGroupName( 470 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 471 parser.addArgument(initialSequentialValue); 472 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 473 474 sequentialValueIncrement = new IntegerArgument(null, 475 "sequentialValueIncrement", false, 1, null, 476 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 477 sequentialAttribute.getIdentifierString())); 478 sequentialValueIncrement.addLongIdentifier("sequential-value-increment", 479 true); 480 sequentialValueIncrement.setArgumentGroupName( 481 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 482 parser.addArgument(sequentialValueIncrement); 483 parser.addDependentArgumentSet(sequentialValueIncrement, 484 sequentialAttribute); 485 486 textBeforeSequentialValue = new StringArgument(null, 487 "textBeforeSequentialValue", false, 1, null, 488 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 489 sequentialAttribute.getIdentifierString())); 490 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value", 491 true); 492 textBeforeSequentialValue.setArgumentGroupName( 493 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 494 parser.addArgument(textBeforeSequentialValue); 495 parser.addDependentArgumentSet(textBeforeSequentialValue, 496 sequentialAttribute); 497 498 textAfterSequentialValue = new StringArgument(null, 499 "textAfterSequentialValue", false, 1, null, 500 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 501 sequentialAttribute.getIdentifierString())); 502 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value", 503 true); 504 textAfterSequentialValue.setArgumentGroupName( 505 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 506 parser.addArgument(textAfterSequentialValue); 507 parser.addDependentArgumentSet(textAfterSequentialValue, 508 sequentialAttribute); 509 510 511 // Add arguments pertaining to attribute value replacement. 512 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 513 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 514 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 515 sourceContainsChangeRecords.getIdentifierString())); 516 replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true); 517 replaceValuesAttribute.setArgumentGroupName( 518 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 519 parser.addArgument(replaceValuesAttribute); 520 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 521 replaceValuesAttribute); 522 523 replacementValue = new StringArgument(null, "replacementValue", false, 0, 524 null, 525 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 526 replaceValuesAttribute.getIdentifierString())); 527 replacementValue.addLongIdentifier("replacement-value", true); 528 replacementValue.setArgumentGroupName( 529 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 530 parser.addArgument(replacementValue); 531 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 532 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 533 534 535 // Add arguments pertaining to adding missing attributes. 536 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 537 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 538 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 539 "--addAttributeValue", 540 sourceContainsChangeRecords.getIdentifierString())); 541 addAttributeName.addLongIdentifier("add-attribute-name", true); 542 addAttributeName.setArgumentGroupName( 543 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 544 parser.addArgument(addAttributeName); 545 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 546 addAttributeName); 547 548 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 549 null, 550 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 551 addAttributeName.getIdentifierString())); 552 addAttributeValue.addLongIdentifier("add-attribute-value", true); 553 addAttributeValue.setArgumentGroupName( 554 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 555 parser.addArgument(addAttributeValue); 556 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 557 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 558 559 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 560 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 561 addAttributeName.getIdentifierString(), 562 addAttributeValue.getIdentifierString())); 563 addToExistingValues.addLongIdentifier("add-to-existing-values", true); 564 addToExistingValues.setArgumentGroupName( 565 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 566 parser.addArgument(addToExistingValues); 567 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 568 569 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 570 null, 571 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 572 addAttributeName.getIdentifierString())); 573 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true); 574 addAttributeBaseDN.setArgumentGroupName( 575 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 576 parser.addArgument(addAttributeBaseDN); 577 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 578 579 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 580 null, 581 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 582 addAttributeBaseDN.getIdentifierString(), 583 addAttributeName.getIdentifierString())); 584 addAttributeScope.addLongIdentifier("add-attribute-scope", true); 585 addAttributeScope.setArgumentGroupName( 586 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 587 parser.addArgument(addAttributeScope); 588 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 589 590 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 591 1, null, 592 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 593 addAttributeName.getIdentifierString())); 594 addAttributeFilter.addLongIdentifier("add-attribute-filter", true); 595 addAttributeFilter.setArgumentGroupName( 596 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 597 parser.addArgument(addAttributeFilter); 598 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 599 600 601 // Add arguments pertaining to renaming attributes. 602 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 603 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 604 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 605 renameAttributeFrom.addLongIdentifier("rename-attribute-from", true); 606 renameAttributeFrom.setArgumentGroupName( 607 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 608 parser.addArgument(renameAttributeFrom); 609 610 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 611 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 612 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 613 renameAttributeFrom.getIdentifierString())); 614 renameAttributeTo.addLongIdentifier("rename-attribute-to", true); 615 renameAttributeTo.setArgumentGroupName( 616 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 617 parser.addArgument(renameAttributeTo); 618 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 619 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 620 621 622 // Add arguments pertaining to flattening subtrees. 623 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 624 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 625 flattenBaseDN.addLongIdentifier("flatten-base-dn", true); 626 flattenBaseDN.setArgumentGroupName( 627 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 628 parser.addArgument(flattenBaseDN); 629 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 630 flattenBaseDN); 631 632 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 633 "flattenAddOmittedRDNAttributesToEntry", 1, 634 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 635 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 636 "flatten-add-omitted-rdn-attributes-to-entry", true); 637 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 638 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 639 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 640 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 641 flattenBaseDN); 642 643 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 644 "flattenAddOmittedRDNAttributesToRDN", 1, 645 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 646 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 647 "flatten-add-omitted-rdn-attributes-to-rdn", true); 648 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 649 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 650 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 651 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 652 flattenBaseDN); 653 654 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 655 false, 1, null, 656 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 657 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true); 658 flattenExcludeFilter.setArgumentGroupName( 659 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 660 parser.addArgument(flattenExcludeFilter); 661 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 662 663 664 // Add arguments pertaining to moving subtrees. 665 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 666 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 667 moveSubtreeFrom.addLongIdentifier("move-subtree-from", true); 668 moveSubtreeFrom.setArgumentGroupName( 669 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 670 parser.addArgument(moveSubtreeFrom); 671 672 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 673 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 674 moveSubtreeFrom.getIdentifierString())); 675 moveSubtreeTo.addLongIdentifier("move-subtree-to", true); 676 moveSubtreeTo.setArgumentGroupName( 677 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 678 parser.addArgument(moveSubtreeTo); 679 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 680 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 681 682 683 // Add arguments pertaining to redacting attribute values. 684 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 685 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 686 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 687 redactAttribute.addLongIdentifier("redact-attribute", true); 688 redactAttribute.setArgumentGroupName( 689 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 690 parser.addArgument(redactAttribute); 691 692 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 693 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 694 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count", 695 true); 696 hideRedactedValueCount.setArgumentGroupName( 697 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 698 parser.addArgument(hideRedactedValueCount); 699 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 700 701 702 // Add arguments pertaining to excluding attributes and entries. 703 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 704 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 705 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 706 excludeAttribute.addLongIdentifier("suppressAttribute", true); 707 excludeAttribute.addLongIdentifier("exclude-attribute", true); 708 excludeAttribute.addLongIdentifier("suppress-attribute", true); 709 excludeAttribute.setArgumentGroupName( 710 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 711 parser.addArgument(excludeAttribute); 712 713 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 714 null, 715 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 716 sourceContainsChangeRecords.getIdentifierString())); 717 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true); 718 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true); 719 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true); 720 excludeEntryBaseDN.setArgumentGroupName( 721 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 722 parser.addArgument(excludeEntryBaseDN); 723 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 724 excludeEntryBaseDN); 725 726 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 727 null, 728 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 729 sourceContainsChangeRecords.getIdentifierString())); 730 excludeEntryScope.addLongIdentifier("suppressEntryScope", true); 731 excludeEntryScope.addLongIdentifier("exclude-entry-scope", true); 732 excludeEntryScope.addLongIdentifier("suppress-entry-scope", true); 733 excludeEntryScope.setArgumentGroupName( 734 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 735 parser.addArgument(excludeEntryScope); 736 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 737 excludeEntryScope); 738 739 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 740 1, null, 741 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 742 sourceContainsChangeRecords.getIdentifierString())); 743 excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true); 744 excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true); 745 excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true); 746 excludeEntryFilter.setArgumentGroupName( 747 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 748 parser.addArgument(excludeEntryFilter); 749 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 750 excludeEntryFilter); 751 752 excludeNonMatchingEntries = new BooleanArgument(null, 753 "excludeNonMatchingEntries", 754 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 755 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries", 756 true); 757 excludeNonMatchingEntries.setArgumentGroupName( 758 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 759 parser.addArgument(excludeNonMatchingEntries); 760 parser.addDependentArgumentSet(excludeNonMatchingEntries, 761 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 762 763 764 // Add arguments for excluding records based on their change types. 765 excludeChangeType = new StringArgument(null, "excludeChangeType", 766 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(), 767 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(), 768 StaticUtils.setOf("add", "delete", "modify", "moddn")); 769 excludeChangeType.addLongIdentifier("exclude-change-type", true); 770 excludeChangeType.addLongIdentifier("exclude-changetype", true); 771 excludeChangeType.setArgumentGroupName( 772 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 773 parser.addArgument(excludeChangeType); 774 775 776 // Add arguments for excluding records that don't have a change type. 777 excludeRecordsWithoutChangeType = new BooleanArgument(null, 778 "excludeRecordsWithoutChangeType", 1, 779 INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get()); 780 excludeRecordsWithoutChangeType.addLongIdentifier( 781 "exclude-records-without-change-type", true); 782 excludeRecordsWithoutChangeType.addLongIdentifier( 783 "exclude-records-without-changetype", true); 784 excludeRecordsWithoutChangeType.setArgumentGroupName( 785 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 786 parser.addArgument(excludeRecordsWithoutChangeType); 787 788 789 // Add the remaining arguments. 790 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 791 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 792 true, true, false, false); 793 schemaPath.addLongIdentifier("schemaFile", true); 794 schemaPath.addLongIdentifier("schemaDirectory", true); 795 schemaPath.addLongIdentifier("schema-path", true); 796 schemaPath.addLongIdentifier("schema-file", true); 797 schemaPath.addLongIdentifier("schema-directory", true); 798 parser.addArgument(schemaPath); 799 800 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 801 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 802 1); 803 numThreads.addLongIdentifier("num-threads", true); 804 parser.addArgument(numThreads); 805 806 processDNs = new BooleanArgument('d', "processDNs", 807 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 808 processDNs.addLongIdentifier("process-dns", true); 809 parser.addArgument(processDNs); 810 811 812 // Ensure that at least one kind of transformation was requested. 813 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 814 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 815 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 816 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter, 817 excludeChangeType, excludeRecordsWithoutChangeType); 818 } 819 820 821 822 /** 823 * {@inheritDoc} 824 */ 825 @Override() 826 public void doExtendedArgumentValidation() 827 throws ArgumentException 828 { 829 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 830 // arguments should always be provided. But in order to preserve backward 831 // compatibility with a legacy scramble-ldif tool, we will allow both to be 832 // omitted if either --scrambleAttribute or --sequentialArgument is 833 // provided. In that case, the path of the output file will be the path of 834 // the first input file with ".scrambled" appended to it. 835 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 836 { 837 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 838 { 839 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 840 targetLDIF.getIdentifierString(), 841 targetToStandardOutput.getIdentifierString())); 842 } 843 } 844 845 846 // Make sure that the --renameAttributeFrom and --renameAttributeTo 847 // arguments were provided an equal number of times. 848 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 849 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 850 if (renameFromOccurrences != renameToOccurrences) 851 { 852 throw new ArgumentException( 853 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 854 renameAttributeFrom.getIdentifierString(), 855 renameAttributeTo.getIdentifierString())); 856 } 857 858 859 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 860 // provided an equal number of times. 861 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 862 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 863 if (moveFromOccurrences != moveToOccurrences) 864 { 865 throw new ArgumentException( 866 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 867 moveSubtreeFrom.getIdentifierString(), 868 moveSubtreeTo.getIdentifierString())); 869 } 870 } 871 872 873 874 /** 875 * {@inheritDoc} 876 */ 877 @Override() 878 public ResultCode doToolProcessing() 879 { 880 final Schema schema; 881 try 882 { 883 schema = getSchema(); 884 } 885 catch (final LDAPException le) 886 { 887 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 888 return le.getResultCode(); 889 } 890 891 892 // If an encryption passphrase file is provided, then get the passphrase 893 // from it. 894 String encryptionPassphrase = null; 895 if (encryptionPassphraseFile.isPresent()) 896 { 897 try 898 { 899 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 900 encryptionPassphraseFile.getValue()); 901 } 902 catch (final LDAPException e) 903 { 904 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage()); 905 return e.getResultCode(); 906 } 907 } 908 909 910 // Create the translators to use to apply the transformations. 911 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 912 new ArrayList<>(10); 913 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 914 new ArrayList<>(10); 915 916 final AtomicLong excludedEntryCount = new AtomicLong(0L); 917 createTranslators(entryTranslators, changeRecordTranslators, 918 schema, excludedEntryCount); 919 920 final AggregateLDIFReaderEntryTranslator entryTranslator = 921 new AggregateLDIFReaderEntryTranslator(entryTranslators); 922 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 923 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 924 925 926 // Determine the path to the target file to be written. 927 final File targetFile; 928 if (targetLDIF.isPresent()) 929 { 930 targetFile = targetLDIF.getValue(); 931 } 932 else if (targetToStandardOutput.isPresent()) 933 { 934 targetFile = null; 935 } 936 else 937 { 938 targetFile = 939 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 940 } 941 942 943 // Create the LDIF reader. 944 final LDIFReader ldifReader; 945 try 946 { 947 final InputStream inputStream; 948 if (sourceLDIF.isPresent()) 949 { 950 final ObjectPair<InputStream,String> p = 951 ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(), 952 encryptionPassphrase, getOut(), getErr()); 953 inputStream = p.getFirst(); 954 if ((encryptionPassphrase == null) && (p.getSecond() != null)) 955 { 956 encryptionPassphrase = p.getSecond(); 957 } 958 } 959 else 960 { 961 inputStream = System.in; 962 } 963 964 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 965 entryTranslator, changeRecordTranslator); 966 if (schema != null) 967 { 968 ldifReader.setSchema(schema); 969 } 970 } 971 catch (final Exception e) 972 { 973 Debug.debugException(e); 974 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 975 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 976 StaticUtils.getExceptionMessage(e))); 977 return ResultCode.LOCAL_ERROR; 978 } 979 980 981 ResultCode resultCode = ResultCode.SUCCESS; 982 OutputStream outputStream = null; 983processingBlock: 984 try 985 { 986 // Create the output stream to use to write the transformed data. 987 try 988 { 989 if (targetFile == null) 990 { 991 outputStream = getOut(); 992 } 993 else 994 { 995 outputStream = 996 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 997 } 998 999 if (encryptTarget.isPresent()) 1000 { 1001 if (encryptionPassphrase == null) 1002 { 1003 encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase( 1004 false, true, getOut(), getErr()); 1005 } 1006 1007 outputStream = new PassphraseEncryptedOutputStream( 1008 encryptionPassphrase, outputStream); 1009 } 1010 1011 if (compressTarget.isPresent()) 1012 { 1013 outputStream = new GZIPOutputStream(outputStream); 1014 } 1015 } 1016 catch (final Exception e) 1017 { 1018 Debug.debugException(e); 1019 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1020 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 1021 targetFile.getAbsolutePath(), 1022 StaticUtils.getExceptionMessage(e))); 1023 resultCode = ResultCode.LOCAL_ERROR; 1024 break processingBlock; 1025 } 1026 1027 1028 // Read the source data one record at a time. The transformations will 1029 // automatically be applied by the LDIF reader's translators, and even if 1030 // there are multiple reader threads, we're guaranteed to get the results 1031 // in the right order. 1032 long entriesWritten = 0L; 1033 while (true) 1034 { 1035 final LDIFRecord ldifRecord; 1036 try 1037 { 1038 ldifRecord = ldifReader.readLDIFRecord(); 1039 } 1040 catch (final LDIFException le) 1041 { 1042 Debug.debugException(le); 1043 if (le.mayContinueReading()) 1044 { 1045 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1046 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 1047 StaticUtils.getExceptionMessage(le))); 1048 if (resultCode == ResultCode.SUCCESS) 1049 { 1050 resultCode = ResultCode.PARAM_ERROR; 1051 } 1052 continue; 1053 } 1054 else 1055 { 1056 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1057 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 1058 StaticUtils.getExceptionMessage(le))); 1059 if (resultCode == ResultCode.SUCCESS) 1060 { 1061 resultCode = ResultCode.PARAM_ERROR; 1062 } 1063 break processingBlock; 1064 } 1065 } 1066 catch (final Exception e) 1067 { 1068 Debug.debugException(e); 1069 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1070 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1071 StaticUtils.getExceptionMessage(e))); 1072 resultCode = ResultCode.LOCAL_ERROR; 1073 break processingBlock; 1074 } 1075 1076 1077 // If the LDIF record is null, then we've run out of records so we're 1078 // done. 1079 if (ldifRecord == null) 1080 { 1081 break; 1082 } 1083 1084 1085 // Write the record to the output stream. 1086 try 1087 { 1088 if (ldifRecord instanceof PreEncodedLDIFEntry) 1089 { 1090 outputStream.write( 1091 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1092 } 1093 else 1094 { 1095 final ByteStringBuffer buffer = getBuffer(); 1096 if (wrapColumn.isPresent()) 1097 { 1098 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1099 } 1100 else 1101 { 1102 ldifRecord.toLDIF(buffer, 0); 1103 } 1104 buffer.append(StaticUtils.EOL_BYTES); 1105 buffer.write(outputStream); 1106 } 1107 } 1108 catch (final Exception e) 1109 { 1110 Debug.debugException(e); 1111 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1112 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1113 StaticUtils.getExceptionMessage(e))); 1114 resultCode = ResultCode.LOCAL_ERROR; 1115 break processingBlock; 1116 } 1117 1118 1119 // If we've written a multiple of 1000 entries, print a progress 1120 // message. 1121 entriesWritten++; 1122 if ((! targetToStandardOutput.isPresent()) && 1123 ((entriesWritten % 1000L) == 0)) 1124 { 1125 final long numExcluded = excludedEntryCount.get(); 1126 if (numExcluded > 0L) 1127 { 1128 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1129 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1130 entriesWritten, numExcluded)); 1131 } 1132 else 1133 { 1134 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1135 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1136 entriesWritten)); 1137 } 1138 } 1139 } 1140 1141 1142 if (! targetToStandardOutput.isPresent()) 1143 { 1144 final long numExcluded = excludedEntryCount.get(); 1145 if (numExcluded > 0L) 1146 { 1147 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1148 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1149 numExcluded)); 1150 } 1151 else 1152 { 1153 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1154 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1155 } 1156 } 1157 } 1158 finally 1159 { 1160 if (outputStream != null) 1161 { 1162 try 1163 { 1164 outputStream.close(); 1165 } 1166 catch (final Exception e) 1167 { 1168 Debug.debugException(e); 1169 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1170 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1171 targetFile.getAbsolutePath(), 1172 StaticUtils.getExceptionMessage(e))); 1173 if (resultCode == ResultCode.SUCCESS) 1174 { 1175 resultCode = ResultCode.LOCAL_ERROR; 1176 } 1177 } 1178 } 1179 1180 try 1181 { 1182 ldifReader.close(); 1183 } 1184 catch (final Exception e) 1185 { 1186 Debug.debugException(e); 1187 // We can ignore this. 1188 } 1189 } 1190 1191 1192 return resultCode; 1193 } 1194 1195 1196 1197 /** 1198 * Retrieves the schema that should be used for processing. 1199 * 1200 * @return The schema that was created. 1201 * 1202 * @throws LDAPException If a problem is encountered while retrieving the 1203 * schema. 1204 */ 1205 private Schema getSchema() 1206 throws LDAPException 1207 { 1208 // If any schema paths were specified, then load the schema only from those 1209 // paths. 1210 if (schemaPath.isPresent()) 1211 { 1212 final ArrayList<File> schemaFiles = new ArrayList<>(10); 1213 for (final File path : schemaPath.getValues()) 1214 { 1215 if (path.isFile()) 1216 { 1217 schemaFiles.add(path); 1218 } 1219 else 1220 { 1221 final TreeMap<String,File> fileMap = new TreeMap<>(); 1222 for (final File schemaDirFile : path.listFiles()) 1223 { 1224 final String name = schemaDirFile.getName(); 1225 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1226 { 1227 fileMap.put(name, schemaDirFile); 1228 } 1229 } 1230 schemaFiles.addAll(fileMap.values()); 1231 } 1232 } 1233 1234 if (schemaFiles.isEmpty()) 1235 { 1236 throw new LDAPException(ResultCode.PARAM_ERROR, 1237 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1238 schemaPath.getIdentifierString())); 1239 } 1240 else 1241 { 1242 try 1243 { 1244 return Schema.getSchema(schemaFiles); 1245 } 1246 catch (final Exception e) 1247 { 1248 Debug.debugException(e); 1249 throw new LDAPException(ResultCode.LOCAL_ERROR, 1250 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1251 StaticUtils.getExceptionMessage(e))); 1252 } 1253 } 1254 } 1255 else 1256 { 1257 // If the INSTANCE_ROOT environment variable is set and it refers to a 1258 // directory that has a config/schema subdirectory that has one or more 1259 // schema files in it, then read the schema from that directory. 1260 try 1261 { 1262 final String instanceRootStr = 1263 StaticUtils.getEnvironmentVariable("INSTANCE_ROOT"); 1264 if (instanceRootStr != null) 1265 { 1266 final File instanceRoot = new File(instanceRootStr); 1267 final File configDir = new File(instanceRoot, "config"); 1268 final File schemaDir = new File(configDir, "schema"); 1269 if (schemaDir.exists()) 1270 { 1271 final TreeMap<String,File> fileMap = new TreeMap<>(); 1272 for (final File schemaDirFile : schemaDir.listFiles()) 1273 { 1274 final String name = schemaDirFile.getName(); 1275 if (schemaDirFile.isFile() && 1276 name.toLowerCase().endsWith(".ldif")) 1277 { 1278 fileMap.put(name, schemaDirFile); 1279 } 1280 } 1281 1282 if (! fileMap.isEmpty()) 1283 { 1284 return Schema.getSchema(new ArrayList<>(fileMap.values())); 1285 } 1286 } 1287 } 1288 } 1289 catch (final Exception e) 1290 { 1291 Debug.debugException(e); 1292 } 1293 } 1294 1295 1296 // If we've gotten here, then just return null and the tool will try to use 1297 // the default standard schema. 1298 return null; 1299 } 1300 1301 1302 1303 /** 1304 * Creates the entry and change record translators that will be used to 1305 * perform the transformations. 1306 * 1307 * @param entryTranslators A list to which all created entry 1308 * translators should be written. 1309 * @param changeRecordTranslators A list to which all created change record 1310 * translators should be written. 1311 * @param schema The schema to use when processing. 1312 * @param excludedEntryCount A counter used to keep track of the number 1313 * of entries that have been excluded from 1314 * the result set. 1315 */ 1316 private void createTranslators( 1317 final List<LDIFReaderEntryTranslator> entryTranslators, 1318 final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators, 1319 final Schema schema, final AtomicLong excludedEntryCount) 1320 { 1321 if (scrambleAttribute.isPresent()) 1322 { 1323 final Long seed; 1324 if (randomSeed.isPresent()) 1325 { 1326 seed = randomSeed.getValue().longValue(); 1327 } 1328 else 1329 { 1330 seed = null; 1331 } 1332 1333 final ScrambleAttributeTransformation t = 1334 new ScrambleAttributeTransformation(schema, seed, 1335 processDNs.isPresent(), scrambleAttribute.getValues(), 1336 scrambleJSONField.getValues()); 1337 entryTranslators.add(t); 1338 changeRecordTranslators.add(t); 1339 } 1340 1341 if (sequentialAttribute.isPresent()) 1342 { 1343 final long initialValue; 1344 if (initialSequentialValue.isPresent()) 1345 { 1346 initialValue = initialSequentialValue.getValue().longValue(); 1347 } 1348 else 1349 { 1350 initialValue = 0L; 1351 } 1352 1353 final long incrementAmount; 1354 if (sequentialValueIncrement.isPresent()) 1355 { 1356 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1357 } 1358 else 1359 { 1360 incrementAmount = 1L; 1361 } 1362 1363 for (final String attrName : sequentialAttribute.getValues()) 1364 { 1365 1366 1367 final ReplaceWithCounterTransformation t = 1368 new ReplaceWithCounterTransformation(schema, attrName, 1369 initialValue, incrementAmount, 1370 textBeforeSequentialValue.getValue(), 1371 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1372 entryTranslators.add(t); 1373 } 1374 } 1375 1376 if (replaceValuesAttribute.isPresent()) 1377 { 1378 final ReplaceAttributeTransformation t = 1379 new ReplaceAttributeTransformation(schema, 1380 replaceValuesAttribute.getValue(), 1381 replacementValue.getValues()); 1382 entryTranslators.add(t); 1383 } 1384 1385 if (addAttributeName.isPresent()) 1386 { 1387 final AddAttributeTransformation t = new AddAttributeTransformation( 1388 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1389 addAttributeFilter.getValue(), 1390 new Attribute(addAttributeName.getValue(), schema, 1391 addAttributeValue.getValues()), 1392 (! addToExistingValues.isPresent())); 1393 entryTranslators.add(t); 1394 } 1395 1396 if (renameAttributeFrom.isPresent()) 1397 { 1398 final Iterator<String> renameFromIterator = 1399 renameAttributeFrom.getValues().iterator(); 1400 final Iterator<String> renameToIterator = 1401 renameAttributeTo.getValues().iterator(); 1402 while (renameFromIterator.hasNext()) 1403 { 1404 final RenameAttributeTransformation t = 1405 new RenameAttributeTransformation(schema, 1406 renameFromIterator.next(), renameToIterator.next(), 1407 processDNs.isPresent()); 1408 entryTranslators.add(t); 1409 changeRecordTranslators.add(t); 1410 } 1411 } 1412 1413 if (flattenBaseDN.isPresent()) 1414 { 1415 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1416 schema, flattenBaseDN.getValue(), 1417 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1418 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1419 flattenExcludeFilter.getValue()); 1420 entryTranslators.add(t); 1421 } 1422 1423 if (moveSubtreeFrom.isPresent()) 1424 { 1425 final Iterator<DN> moveFromIterator = 1426 moveSubtreeFrom.getValues().iterator(); 1427 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1428 while (moveFromIterator.hasNext()) 1429 { 1430 final MoveSubtreeTransformation t = 1431 new MoveSubtreeTransformation(moveFromIterator.next(), 1432 moveToIterator.next()); 1433 entryTranslators.add(t); 1434 changeRecordTranslators.add(t); 1435 } 1436 } 1437 1438 if (redactAttribute.isPresent()) 1439 { 1440 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1441 schema, processDNs.isPresent(), 1442 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1443 entryTranslators.add(t); 1444 changeRecordTranslators.add(t); 1445 } 1446 1447 if (excludeAttribute.isPresent()) 1448 { 1449 final ExcludeAttributeTransformation t = 1450 new ExcludeAttributeTransformation(schema, 1451 excludeAttribute.getValues()); 1452 entryTranslators.add(t); 1453 changeRecordTranslators.add(t); 1454 } 1455 1456 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1457 excludeEntryFilter.isPresent()) 1458 { 1459 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1460 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1461 excludeEntryFilter.getValue(), 1462 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1463 entryTranslators.add(t); 1464 } 1465 1466 if (excludeChangeType.isPresent()) 1467 { 1468 final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class); 1469 for (final String changeTypeName : excludeChangeType.getValues()) 1470 { 1471 changeTypes.add(ChangeType.forName(changeTypeName)); 1472 } 1473 1474 changeRecordTranslators.add( 1475 new ExcludeChangeTypeTransformation(changeTypes)); 1476 } 1477 1478 if (excludeRecordsWithoutChangeType.isPresent()) 1479 { 1480 entryTranslators.add(new ExcludeAllEntriesTransformation()); 1481 } 1482 1483 entryTranslators.add(this); 1484 } 1485 1486 1487 1488 /** 1489 * {@inheritDoc} 1490 */ 1491 @Override() 1492 public LinkedHashMap<String[],String> getExampleUsages() 1493 { 1494 final LinkedHashMap<String[],String> examples = 1495 new LinkedHashMap<>(StaticUtils.computeMapCapacity(4)); 1496 1497 examples.put( 1498 new String[] 1499 { 1500 "--sourceLDIF", "input.ldif", 1501 "--targetLDIF", "scrambled.ldif", 1502 "--scrambleAttribute", "givenName", 1503 "--scrambleAttribute", "sn", 1504 "--scrambleAttribute", "cn", 1505 "--numThreads", "10", 1506 "--schemaPath", "/ds/config/schema", 1507 "--processDNs" 1508 }, 1509 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1510 1511 examples.put( 1512 new String[] 1513 { 1514 "--sourceLDIF", "input.ldif", 1515 "--targetLDIF", "sequential.ldif", 1516 "--sequentialAttribute", "uid", 1517 "--initialSequentialValue", "1", 1518 "--sequentialValueIncrement", "1", 1519 "--textBeforeSequentialValue", "user.", 1520 "--numThreads", "10", 1521 "--schemaPath", "/ds/config/schema", 1522 "--processDNs" 1523 }, 1524 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1525 1526 examples.put( 1527 new String[] 1528 { 1529 "--sourceLDIF", "input.ldif", 1530 "--targetLDIF", "added-organization.ldif", 1531 "--addAttributeName", "o", 1532 "--addAttributeValue", "Example Corp.", 1533 "--addAttributeFilter", "(objectClass=person)", 1534 "--numThreads", "10", 1535 "--schemaPath", "/ds/config/schema" 1536 }, 1537 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1538 1539 examples.put( 1540 new String[] 1541 { 1542 "--sourceLDIF", "input.ldif", 1543 "--targetLDIF", "rebased.ldif", 1544 "--moveSubtreeFrom", "o=example.com", 1545 "--moveSubtreeTo", "dc=example,dc=com", 1546 "--numThreads", "10", 1547 "--schemaPath", "/ds/config/schema" 1548 }, 1549 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1550 1551 return examples; 1552 } 1553 1554 1555 1556 /** 1557 * {@inheritDoc} 1558 */ 1559 @Override() 1560 public Entry translate(final Entry original, final long firstLineNumber) 1561 throws LDIFException 1562 { 1563 final ByteStringBuffer buffer = getBuffer(); 1564 if (wrapColumn.isPresent()) 1565 { 1566 original.toLDIF(buffer, wrapColumn.getValue()); 1567 } 1568 else 1569 { 1570 original.toLDIF(buffer, 0); 1571 } 1572 buffer.append(StaticUtils.EOL_BYTES); 1573 1574 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1575 } 1576 1577 1578 1579 /** 1580 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1581 * 1582 * @return A byte string buffer that can be used to perform LDIF encoding. 1583 */ 1584 private ByteStringBuffer getBuffer() 1585 { 1586 ByteStringBuffer buffer = byteStringBuffers.get(); 1587 if (buffer == null) 1588 { 1589 buffer = new ByteStringBuffer(); 1590 byteStringBuffers.set(buffer); 1591 } 1592 else 1593 { 1594 buffer.clear(); 1595 } 1596 1597 return buffer; 1598 } 1599}