Bug Summary

File:home/bhubbard/working/src/ceph/src/rocksdb/tools/ldb_cmd.cc
Warning:line 448, column 10
Called C++ object pointer is null

Annotated Source Code

[?] Use j/k keys for keyboard navigation

1// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
2// This source code is licensed under both the GPLv2 (found in the
3// COPYING file in the root directory) and Apache 2.0 License
4// (found in the LICENSE.Apache file in the root directory).
5//
6#ifndef ROCKSDB_LITE
7#include "rocksdb/utilities/ldb_cmd.h"
8
9#ifndef __STDC_FORMAT_MACROS
10#define __STDC_FORMAT_MACROS
11#endif
12
13#include <inttypes.h>
14
15#include "db/db_impl.h"
16#include "db/dbformat.h"
17#include "db/log_reader.h"
18#include "db/write_batch_internal.h"
19#include "port/port_dirent.h"
20#include "rocksdb/cache.h"
21#include "rocksdb/table_properties.h"
22#include "rocksdb/utilities/backupable_db.h"
23#include "rocksdb/utilities/checkpoint.h"
24#include "rocksdb/utilities/debug.h"
25#include "rocksdb/utilities/object_registry.h"
26#include "rocksdb/utilities/options_util.h"
27#include "rocksdb/write_batch.h"
28#include "rocksdb/write_buffer_manager.h"
29#include "table/scoped_arena_iterator.h"
30#include "tools/ldb_cmd_impl.h"
31#include "tools/sst_dump_tool_imp.h"
32#include "util/cast_util.h"
33#include "util/coding.h"
34#include "util/filename.h"
35#include "util/stderr_logger.h"
36#include "util/string_util.h"
37#include "utilities/ttl/db_ttl_impl.h"
38
39#include <cstdlib>
40#include <ctime>
41#include <fstream>
42#include <functional>
43#include <iostream>
44#include <limits>
45#include <sstream>
46#include <stdexcept>
47#include <string>
48
49namespace rocksdb {
50
51const std::string LDBCommand::ARG_DB = "db";
52const std::string LDBCommand::ARG_PATH = "path";
53const std::string LDBCommand::ARG_HEX = "hex";
54const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
55const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
56const std::string LDBCommand::ARG_CF_NAME = "column_family";
57const std::string LDBCommand::ARG_TTL = "ttl";
58const std::string LDBCommand::ARG_TTL_START = "start_time";
59const std::string LDBCommand::ARG_TTL_END = "end_time";
60const std::string LDBCommand::ARG_TIMESTAMP = "timestamp";
61const std::string LDBCommand::ARG_TRY_LOAD_OPTIONS = "try_load_options";
62const std::string LDBCommand::ARG_IGNORE_UNKNOWN_OPTIONS =
63 "ignore_unknown_options";
64const std::string LDBCommand::ARG_FROM = "from";
65const std::string LDBCommand::ARG_TO = "to";
66const std::string LDBCommand::ARG_MAX_KEYS = "max_keys";
67const std::string LDBCommand::ARG_BLOOM_BITS = "bloom_bits";
68const std::string LDBCommand::ARG_FIX_PREFIX_LEN = "fix_prefix_len";
69const std::string LDBCommand::ARG_COMPRESSION_TYPE = "compression_type";
70const std::string LDBCommand::ARG_COMPRESSION_MAX_DICT_BYTES =
71 "compression_max_dict_bytes";
72const std::string LDBCommand::ARG_BLOCK_SIZE = "block_size";
73const std::string LDBCommand::ARG_AUTO_COMPACTION = "auto_compaction";
74const std::string LDBCommand::ARG_DB_WRITE_BUFFER_SIZE = "db_write_buffer_size";
75const std::string LDBCommand::ARG_WRITE_BUFFER_SIZE = "write_buffer_size";
76const std::string LDBCommand::ARG_FILE_SIZE = "file_size";
77const std::string LDBCommand::ARG_CREATE_IF_MISSING = "create_if_missing";
78const std::string LDBCommand::ARG_NO_VALUE = "no_value";
79
80const char* LDBCommand::DELIM = " ==> ";
81
82namespace {
83
84void DumpWalFile(Options options, std::string wal_file, bool print_header,
85 bool print_values, bool is_write_committed,
86 LDBCommandExecuteResult* exec_state);
87
88void DumpSstFile(Options options, std::string filename, bool output_hex,
89 bool show_properties);
90};
91
92LDBCommand* LDBCommand::InitFromCmdLineArgs(
93 int argc, char** argv, const Options& options,
94 const LDBOptions& ldb_options,
95 const std::vector<ColumnFamilyDescriptor>* column_families) {
96 std::vector<std::string> args;
97 for (int i = 1; i < argc; i++) {
98 args.push_back(argv[i]);
99 }
100 return InitFromCmdLineArgs(args, options, ldb_options, column_families,
101 SelectCommand);
102}
103
104/**
105 * Parse the command-line arguments and create the appropriate LDBCommand2
106 * instance.
107 * The command line arguments must be in the following format:
108 * ./ldb --db=PATH_TO_DB [--commonOpt1=commonOpt1Val] ..
109 * COMMAND <PARAM1> <PARAM2> ... [-cmdSpecificOpt1=cmdSpecificOpt1Val] ..
110 * This is similar to the command line format used by HBaseClientTool.
111 * Command name is not included in args.
112 * Returns nullptr if the command-line cannot be parsed.
113 */
114LDBCommand* LDBCommand::InitFromCmdLineArgs(
115 const std::vector<std::string>& args, const Options& options,
116 const LDBOptions& ldb_options,
117 const std::vector<ColumnFamilyDescriptor>* /*column_families*/,
118 const std::function<LDBCommand*(const ParsedParams&)>& selector) {
119 // --x=y command line arguments are added as x->y map entries in
120 // parsed_params.option_map.
121 //
122 // Command-line arguments of the form --hex end up in this array as hex to
123 // parsed_params.flags
124 ParsedParams parsed_params;
125
126 // Everything other than option_map and flags. Represents commands
127 // and their parameters. For eg: put key1 value1 go into this vector.
128 std::vector<std::string> cmdTokens;
129
130 const std::string OPTION_PREFIX = "--";
131
132 for (const auto& arg : args) {
133 if (arg[0] == '-' && arg[1] == '-'){
134 std::vector<std::string> splits = StringSplit(arg, '=');
135 // --option_name=option_value
136 if (splits.size() == 2) {
137 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
138 parsed_params.option_map[optionKey] = splits[1];
139 } else if (splits.size() == 1) {
140 // --flag_name
141 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
142 parsed_params.flags.push_back(optionKey);
143 } else {
144 // --option_name=option_value, option_value contains '='
145 std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
146 parsed_params.option_map[optionKey] =
147 arg.substr(splits[0].length() + 1);
148 }
149 } else {
150 cmdTokens.push_back(arg);
151 }
152 }
153
154 if (cmdTokens.size() < 1) {
155 fprintf(stderrstderr, "Command not specified!");
156 return nullptr;
157 }
158
159 parsed_params.cmd = cmdTokens[0];
160 parsed_params.cmd_params.assign(cmdTokens.begin() + 1, cmdTokens.end());
161
162 LDBCommand* command = selector(parsed_params);
163
164 if (command) {
165 command->SetDBOptions(options);
166 command->SetLDBOptions(ldb_options);
167 }
168 return command;
169}
170
171LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
172 if (parsed_params.cmd == GetCommand::Name()) {
173 return new GetCommand(parsed_params.cmd_params, parsed_params.option_map,
174 parsed_params.flags);
175 } else if (parsed_params.cmd == PutCommand::Name()) {
176 return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
177 parsed_params.flags);
178 } else if (parsed_params.cmd == BatchPutCommand::Name()) {
179 return new BatchPutCommand(parsed_params.cmd_params,
180 parsed_params.option_map, parsed_params.flags);
181 } else if (parsed_params.cmd == ScanCommand::Name()) {
182 return new ScanCommand(parsed_params.cmd_params, parsed_params.option_map,
183 parsed_params.flags);
184 } else if (parsed_params.cmd == DeleteCommand::Name()) {
185 return new DeleteCommand(parsed_params.cmd_params, parsed_params.option_map,
186 parsed_params.flags);
187 } else if (parsed_params.cmd == DeleteRangeCommand::Name()) {
188 return new DeleteRangeCommand(parsed_params.cmd_params,
189 parsed_params.option_map,
190 parsed_params.flags);
191 } else if (parsed_params.cmd == ApproxSizeCommand::Name()) {
192 return new ApproxSizeCommand(parsed_params.cmd_params,
193 parsed_params.option_map, parsed_params.flags);
194 } else if (parsed_params.cmd == DBQuerierCommand::Name()) {
195 return new DBQuerierCommand(parsed_params.cmd_params,
196 parsed_params.option_map, parsed_params.flags);
197 } else if (parsed_params.cmd == CompactorCommand::Name()) {
198 return new CompactorCommand(parsed_params.cmd_params,
199 parsed_params.option_map, parsed_params.flags);
200 } else if (parsed_params.cmd == WALDumperCommand::Name()) {
201 return new WALDumperCommand(parsed_params.cmd_params,
202 parsed_params.option_map, parsed_params.flags);
203 } else if (parsed_params.cmd == ReduceDBLevelsCommand::Name()) {
204 return new ReduceDBLevelsCommand(parsed_params.cmd_params,
205 parsed_params.option_map,
206 parsed_params.flags);
207 } else if (parsed_params.cmd == ChangeCompactionStyleCommand::Name()) {
208 return new ChangeCompactionStyleCommand(parsed_params.cmd_params,
209 parsed_params.option_map,
210 parsed_params.flags);
211 } else if (parsed_params.cmd == DBDumperCommand::Name()) {
212 return new DBDumperCommand(parsed_params.cmd_params,
213 parsed_params.option_map, parsed_params.flags);
214 } else if (parsed_params.cmd == DBLoaderCommand::Name()) {
215 return new DBLoaderCommand(parsed_params.cmd_params,
216 parsed_params.option_map, parsed_params.flags);
217 } else if (parsed_params.cmd == ManifestDumpCommand::Name()) {
218 return new ManifestDumpCommand(parsed_params.cmd_params,
219 parsed_params.option_map,
220 parsed_params.flags);
221 } else if (parsed_params.cmd == ListColumnFamiliesCommand::Name()) {
222 return new ListColumnFamiliesCommand(parsed_params.cmd_params,
223 parsed_params.option_map,
224 parsed_params.flags);
225 } else if (parsed_params.cmd == CreateColumnFamilyCommand::Name()) {
226 return new CreateColumnFamilyCommand(parsed_params.cmd_params,
227 parsed_params.option_map,
228 parsed_params.flags);
229 } else if (parsed_params.cmd == DBFileDumperCommand::Name()) {
230 return new DBFileDumperCommand(parsed_params.cmd_params,
231 parsed_params.option_map,
232 parsed_params.flags);
233 } else if (parsed_params.cmd == InternalDumpCommand::Name()) {
234 return new InternalDumpCommand(parsed_params.cmd_params,
235 parsed_params.option_map,
236 parsed_params.flags);
237 } else if (parsed_params.cmd == CheckConsistencyCommand::Name()) {
238 return new CheckConsistencyCommand(parsed_params.cmd_params,
239 parsed_params.option_map,
240 parsed_params.flags);
241 } else if (parsed_params.cmd == CheckPointCommand::Name()) {
242 return new CheckPointCommand(parsed_params.cmd_params,
243 parsed_params.option_map,
244 parsed_params.flags);
245 } else if (parsed_params.cmd == RepairCommand::Name()) {
246 return new RepairCommand(parsed_params.cmd_params, parsed_params.option_map,
247 parsed_params.flags);
248 } else if (parsed_params.cmd == BackupCommand::Name()) {
249 return new BackupCommand(parsed_params.cmd_params, parsed_params.option_map,
250 parsed_params.flags);
251 } else if (parsed_params.cmd == RestoreCommand::Name()) {
252 return new RestoreCommand(parsed_params.cmd_params,
253 parsed_params.option_map, parsed_params.flags);
254 } else if (parsed_params.cmd == WriteExternalSstFilesCommand::Name()) {
255 return new WriteExternalSstFilesCommand(parsed_params.cmd_params,
256 parsed_params.option_map,
257 parsed_params.flags);
258 } else if (parsed_params.cmd == IngestExternalSstFilesCommand::Name()) {
259 return new IngestExternalSstFilesCommand(parsed_params.cmd_params,
260 parsed_params.option_map,
261 parsed_params.flags);
262 }
263 return nullptr;
264}
265
266/* Run the command, and return the execute result. */
267void LDBCommand::Run() {
268 if (!exec_state_.IsNotStarted()) {
269 return;
270 }
271
272 if (db_ == nullptr && !NoDBOpen()) {
273 OpenDB();
274 if (exec_state_.IsFailed() && try_load_options_) {
275 // We don't always return if there is a failure because a WAL file or
276 // manifest file can be given to "dump" command so we should continue.
277 // --try_load_options is not valid in those cases.
278 return;
279 }
280 }
281
282 // We'll intentionally proceed even if the DB can't be opened because users
283 // can also specify a filename, not just a directory.
284 DoCommand();
285
286 if (exec_state_.IsNotStarted()) {
287 exec_state_ = LDBCommandExecuteResult::Succeed("");
288 }
289
290 if (db_ != nullptr) {
291 CloseDB();
292 }
293}
294
295LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
296 const std::vector<std::string>& flags, bool is_read_only,
297 const std::vector<std::string>& valid_cmd_line_options)
298 : db_(nullptr),
299 db_ttl_(nullptr),
300 is_read_only_(is_read_only),
301 is_key_hex_(false),
302 is_value_hex_(false),
303 is_db_ttl_(false),
304 timestamp_(false),
305 try_load_options_(false),
306 ignore_unknown_options_(false),
307 create_if_missing_(false),
308 option_map_(options),
309 flags_(flags),
310 valid_cmd_line_options_(valid_cmd_line_options) {
311 std::map<std::string, std::string>::const_iterator itr = options.find(ARG_DB);
312 if (itr != options.end()) {
313 db_path_ = itr->second;
314 }
315
316 itr = options.find(ARG_CF_NAME);
317 if (itr != options.end()) {
318 column_family_name_ = itr->second;
319 } else {
320 column_family_name_ = kDefaultColumnFamilyName;
321 }
322
323 is_key_hex_ = IsKeyHex(options, flags);
324 is_value_hex_ = IsValueHex(options, flags);
325 is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
326 timestamp_ = IsFlagPresent(flags, ARG_TIMESTAMP);
327 try_load_options_ = IsFlagPresent(flags, ARG_TRY_LOAD_OPTIONS);
328 ignore_unknown_options_ = IsFlagPresent(flags, ARG_IGNORE_UNKNOWN_OPTIONS);
329}
330
331void LDBCommand::OpenDB() {
332 if (!create_if_missing_ && try_load_options_) {
8
←
Assuming the condition is true
→
9
←
Assuming the condition is true
→
10
←
Taking true branch
→
333 Status s = LoadLatestOptions(db_path_, Env::Default(), &options_,
334 &column_families_, ignore_unknown_options_);
335 if (!s.ok() && !s.IsNotFound()) {
11
←
Taking true branch
→
336 // Option file exists but load option file error.
337 std::string msg = s.ToString();
338 exec_state_ = LDBCommandExecuteResult::Failed(msg);
339 db_ = nullptr;
12
←
Null pointer value stored to field 'db_'
→
340 return;
341 }
342 if (options_.env->FileExists(options_.wal_dir).IsNotFound()) {
343 options_.wal_dir = db_path_;
344 fprintf(
345 stderrstderr,
346 "wal_dir loaded from the option file doesn't exist. Ignore it.\n");
347 }
348 }
349 options_ = PrepareOptionsForOpenDB();
350 if (!exec_state_.IsNotStarted()) {
351 return;
352 }
353 // Open the DB.
354 Status st;
355 std::vector<ColumnFamilyHandle*> handles_opened;
356 if (is_db_ttl_) {
357 // ldb doesn't yet support TTL DB with multiple column families
358 if (!column_family_name_.empty() || !column_families_.empty()) {
359 exec_state_ = LDBCommandExecuteResult::Failed(
360 "ldb doesn't support TTL DB with multiple column families");
361 }
362 if (is_read_only_) {
363 st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
364 } else {
365 st = DBWithTTL::Open(options_, db_path_, &db_ttl_);
366 }
367 db_ = db_ttl_;
368 } else {
369 if (column_families_.empty()) {
370 // Try to figure out column family lists
371 std::vector<std::string> cf_list;
372 st = DB::ListColumnFamilies(DBOptions(), db_path_, &cf_list);
373 // There is possible the DB doesn't exist yet, for "create if not
374 // "existing case". The failure is ignored here. We rely on DB::Open()
375 // to give us the correct error message for problem with opening
376 // existing DB.
377 if (st.ok() && cf_list.size() > 1) {
378 // Ignore single column family DB.
379 for (auto cf_name : cf_list) {
380 column_families_.emplace_back(cf_name, options_);
381 }
382 }
383 }
384 if (is_read_only_) {
385 if (column_families_.empty()) {
386 st = DB::OpenForReadOnly(options_, db_path_, &db_);
387 } else {
388 st = DB::OpenForReadOnly(options_, db_path_, column_families_,
389 &handles_opened, &db_);
390 }
391 } else {
392 if (column_families_.empty()) {
393 st = DB::Open(options_, db_path_, &db_);
394 } else {
395 st = DB::Open(options_, db_path_, column_families_, &handles_opened,
396 &db_);
397 }
398 }
399 }
400 if (!st.ok()) {
401 std::string msg = st.ToString();
402 exec_state_ = LDBCommandExecuteResult::Failed(msg);
403 } else if (!handles_opened.empty()) {
404 assert(handles_opened.size() == column_families_.size())(static_cast<void> (0));
405 bool found_cf_name = false;
406 for (size_t i = 0; i < handles_opened.size(); i++) {
407 cf_handles_[column_families_[i].name] = handles_opened[i];
408 if (column_family_name_ == column_families_[i].name) {
409 found_cf_name = true;
410 }
411 }
412 if (!found_cf_name) {
413 exec_state_ = LDBCommandExecuteResult::Failed(
414 "Non-existing column family " + column_family_name_);
415 CloseDB();
416 }
417 } else {
418 // We successfully opened DB in single column family mode.
419 assert(column_families_.empty())(static_cast<void> (0));
420 if (column_family_name_ != kDefaultColumnFamilyName) {
421 exec_state_ = LDBCommandExecuteResult::Failed(
422 "Non-existing column family " + column_family_name_);
423 CloseDB();
424 }
425 }
426}
427
428void LDBCommand::CloseDB() {
429 if (db_ != nullptr) {
430 for (auto& pair : cf_handles_) {
431 delete pair.second;
432 }
433 delete db_;
434 db_ = nullptr;
435 }
436}
437
438ColumnFamilyHandle* LDBCommand::GetCfHandle() {
439 if (!cf_handles_.empty()) {
16
←
Assuming the condition is false
→
17
←
Taking false branch
→
440 auto it = cf_handles_.find(column_family_name_);
441 if (it == cf_handles_.end()) {
442 exec_state_ = LDBCommandExecuteResult::Failed(
443 "Cannot find column family " + column_family_name_);
444 } else {
445 return it->second;
446 }
447 }
448 return db_->DefaultColumnFamily();
18
←
Called C++ object pointer is null
449}
450
451std::vector<std::string> LDBCommand::BuildCmdLineOptions(
452 std::vector<std::string> options) {
453 std::vector<std::string> ret = {ARG_DB,
454 ARG_BLOOM_BITS,
455 ARG_BLOCK_SIZE,
456 ARG_AUTO_COMPACTION,
457 ARG_COMPRESSION_TYPE,
458 ARG_COMPRESSION_MAX_DICT_BYTES,
459 ARG_WRITE_BUFFER_SIZE,
460 ARG_FILE_SIZE,
461 ARG_FIX_PREFIX_LEN,
462 ARG_TRY_LOAD_OPTIONS,
463 ARG_IGNORE_UNKNOWN_OPTIONS,
464 ARG_CF_NAME};
465 ret.insert(ret.end(), options.begin(), options.end());
466 return ret;
467}
468
469/**
470 * Parses the specific integer option and fills in the value.
471 * Returns true if the option is found.
472 * Returns false if the option is not found or if there is an error parsing the
473 * value. If there is an error, the specified exec_state is also
474 * updated.
475 */
476bool LDBCommand::ParseIntOption(
477 const std::map<std::string, std::string>& /*options*/,
478 const std::string& option, int& value,
479 LDBCommandExecuteResult& exec_state) {
480 std::map<std::string, std::string>::const_iterator itr =
481 option_map_.find(option);
482 if (itr != option_map_.end()) {
483 try {
484#if defined(CYGWIN)
485 value = strtol(itr->second.c_str(), 0, 10);
486#else
487 value = std::stoi(itr->second);
488#endif
489 return true;
490 } catch (const std::invalid_argument&) {
491 exec_state =
492 LDBCommandExecuteResult::Failed(option + " has an invalid value.");
493 } catch (const std::out_of_range&) {
494 exec_state = LDBCommandExecuteResult::Failed(
495 option + " has a value out-of-range.");
496 }
497 }
498 return false;
499}
500
501/**
502 * Parses the specified option and fills in the value.
503 * Returns true if the option is found.
504 * Returns false otherwise.
505 */
506bool LDBCommand::ParseStringOption(
507 const std::map<std::string, std::string>& /*options*/,
508 const std::string& option, std::string* value) {
509 auto itr = option_map_.find(option);
510 if (itr != option_map_.end()) {
511 *value = itr->second;
512 return true;
513 }
514 return false;
515}
516
517Options LDBCommand::PrepareOptionsForOpenDB() {
518 ColumnFamilyOptions* cf_opts;
519 auto column_families_iter =
520 std::find_if(column_families_.begin(), column_families_.end(),
521 [this](const ColumnFamilyDescriptor& cf_desc) {
522 return cf_desc.name == column_family_name_;
523 });
524 if (column_families_iter != column_families_.end()) {
525 cf_opts = &column_families_iter->options;
526 } else {
527 cf_opts = static_cast<ColumnFamilyOptions*>(&options_);
528 }
529 DBOptions* db_opts = static_cast<DBOptions*>(&options_);
530 db_opts->create_if_missing = false;
531
532 std::map<std::string, std::string>::const_iterator itr;
533
534 BlockBasedTableOptions table_options;
535 bool use_table_options = false;
536 int bits;
537 if (ParseIntOption(option_map_, ARG_BLOOM_BITS, bits, exec_state_)) {
538 if (bits > 0) {
539 use_table_options = true;
540 table_options.filter_policy.reset(NewBloomFilterPolicy(bits));
541 } else {
542 exec_state_ =
543 LDBCommandExecuteResult::Failed(ARG_BLOOM_BITS + " must be > 0.");
544 }
545 }
546
547 int block_size;
548 if (ParseIntOption(option_map_, ARG_BLOCK_SIZE, block_size, exec_state_)) {
549 if (block_size > 0) {
550 use_table_options = true;
551 table_options.block_size = block_size;
552 } else {
553 exec_state_ =
554 LDBCommandExecuteResult::Failed(ARG_BLOCK_SIZE + " must be > 0.");
555 }
556 }
557
558 if (use_table_options) {
559 cf_opts->table_factory.reset(NewBlockBasedTableFactory(table_options));
560 }
561
562 itr = option_map_.find(ARG_AUTO_COMPACTION);
563 if (itr != option_map_.end()) {
564 cf_opts->disable_auto_compactions = !StringToBool(itr->second);
565 }
566
567 itr = option_map_.find(ARG_COMPRESSION_TYPE);
568 if (itr != option_map_.end()) {
569 std::string comp = itr->second;
570 if (comp == "no") {
571 cf_opts->compression = kNoCompression;
572 } else if (comp == "snappy") {
573 cf_opts->compression = kSnappyCompression;
574 } else if (comp == "zlib") {
575 cf_opts->compression = kZlibCompression;
576 } else if (comp == "bzip2") {
577 cf_opts->compression = kBZip2Compression;
578 } else if (comp == "lz4") {
579 cf_opts->compression = kLZ4Compression;
580 } else if (comp == "lz4hc") {
581 cf_opts->compression = kLZ4HCCompression;
582 } else if (comp == "xpress") {
583 cf_opts->compression = kXpressCompression;
584 } else if (comp == "zstd") {
585 cf_opts->compression = kZSTD;
586 } else {
587 // Unknown compression.
588 exec_state_ =
589 LDBCommandExecuteResult::Failed("Unknown compression level: " + comp);
590 }
591 }
592
593 int compression_max_dict_bytes;
594 if (ParseIntOption(option_map_, ARG_COMPRESSION_MAX_DICT_BYTES,
595 compression_max_dict_bytes, exec_state_)) {
596 if (compression_max_dict_bytes >= 0) {
597 cf_opts->compression_opts.max_dict_bytes = compression_max_dict_bytes;
598 } else {
599 exec_state_ = LDBCommandExecuteResult::Failed(
600 ARG_COMPRESSION_MAX_DICT_BYTES + " must be >= 0.");
601 }
602 }
603
604 int db_write_buffer_size;
605 if (ParseIntOption(option_map_, ARG_DB_WRITE_BUFFER_SIZE,
606 db_write_buffer_size, exec_state_)) {
607 if (db_write_buffer_size >= 0) {
608 db_opts->db_write_buffer_size = db_write_buffer_size;
609 } else {
610 exec_state_ = LDBCommandExecuteResult::Failed(ARG_DB_WRITE_BUFFER_SIZE +
611 " must be >= 0.");
612 }
613 }
614
615 int write_buffer_size;
616 if (ParseIntOption(option_map_, ARG_WRITE_BUFFER_SIZE, write_buffer_size,
617 exec_state_)) {
618 if (write_buffer_size > 0) {
619 cf_opts->write_buffer_size = write_buffer_size;
620 } else {
621 exec_state_ = LDBCommandExecuteResult::Failed(ARG_WRITE_BUFFER_SIZE +
622 " must be > 0.");
623 }
624 }
625
626 int file_size;
627 if (ParseIntOption(option_map_, ARG_FILE_SIZE, file_size, exec_state_)) {
628 if (file_size > 0) {
629 cf_opts->target_file_size_base = file_size;
630 } else {
631 exec_state_ =
632 LDBCommandExecuteResult::Failed(ARG_FILE_SIZE + " must be > 0.");
633 }
634 }
635
636 if (db_opts->db_paths.size() == 0) {
637 db_opts->db_paths.emplace_back(db_path_,
638 std::numeric_limits<uint64_t>::max());
639 }
640
641 int fix_prefix_len;
642 if (ParseIntOption(option_map_, ARG_FIX_PREFIX_LEN, fix_prefix_len,
643 exec_state_)) {
644 if (fix_prefix_len > 0) {
645 cf_opts->prefix_extractor.reset(
646 NewFixedPrefixTransform(static_cast<size_t>(fix_prefix_len)));
647 } else {
648 exec_state_ =
649 LDBCommandExecuteResult::Failed(ARG_FIX_PREFIX_LEN + " must be > 0.");
650 }
651 }
652 // TODO(ajkr): this return value doesn't reflect the CF options changed, so
653 // subcommands that rely on this won't see the effect of CF-related CLI args.
654 // Such subcommands need to be changed to properly support CFs.
655 return options_;
656}
657
658bool LDBCommand::ParseKeyValue(const std::string& line, std::string* key,
659 std::string* value, bool is_key_hex,
660 bool is_value_hex) {
661 size_t pos = line.find(DELIM);
662 if (pos != std::string::npos) {
663 *key = line.substr(0, pos);
664 *value = line.substr(pos + strlen(DELIM));
665 if (is_key_hex) {
666 *key = HexToString(*key);
667 }
668 if (is_value_hex) {
669 *value = HexToString(*value);
670 }
671 return true;
672 } else {
673 return false;
674 }
675}
676
677/**
678 * Make sure that ONLY the command-line options and flags expected by this
679 * command are specified on the command-line. Extraneous options are usually
680 * the result of user error.
681 * Returns true if all checks pass. Else returns false, and prints an
682 * appropriate error msg to stderr.
683 */
684bool LDBCommand::ValidateCmdLineOptions() {
685 for (std::map<std::string, std::string>::const_iterator itr =
686 option_map_.begin();
687 itr != option_map_.end(); ++itr) {
688 if (std::find(valid_cmd_line_options_.begin(),
689 valid_cmd_line_options_.end(),
690 itr->first) == valid_cmd_line_options_.end()) {
691 fprintf(stderrstderr, "Invalid command-line option %s\n", itr->first.c_str());
692 return false;
693 }
694 }
695
696 for (std::vector<std::string>::const_iterator itr = flags_.begin();
697 itr != flags_.end(); ++itr) {
698 if (std::find(valid_cmd_line_options_.begin(),
699 valid_cmd_line_options_.end(),
700 *itr) == valid_cmd_line_options_.end()) {
701 fprintf(stderrstderr, "Invalid command-line flag %s\n", itr->c_str());
702 return false;
703 }
704 }
705
706 if (!NoDBOpen() && option_map_.find(ARG_DB) == option_map_.end() &&
707 option_map_.find(ARG_PATH) == option_map_.end()) {
708 fprintf(stderrstderr, "Either %s or %s must be specified.\n", ARG_DB.c_str(),
709 ARG_PATH.c_str());
710 return false;
711 }
712
713 return true;
714}
715
716std::string LDBCommand::HexToString(const std::string& str) {
717 std::string result;
718 std::string::size_type len = str.length();
719 if (len < 2 || str[0] != '0' || str[1] != 'x') {
720 fprintf(stderrstderr, "Invalid hex input %s. Must start with 0x\n", str.c_str());
721 throw "Invalid hex input";
722 }
723 if (!Slice(str.data() + 2, len - 2).DecodeHex(&result)) {
724 throw "Invalid hex input";
725 }
726 return result;
727}
728
729std::string LDBCommand::StringToHex(const std::string& str) {
730 std::string result("0x");
731 result.append(Slice(str).ToString(true));
732 return result;
733}
734
735std::string LDBCommand::PrintKeyValue(const std::string& key,
736 const std::string& value, bool is_key_hex,
737 bool is_value_hex) {
738 std::string result;
739 result.append(is_key_hex ? StringToHex(key) : key);
740 result.append(DELIM);
741 result.append(is_value_hex ? StringToHex(value) : value);
742 return result;
743}
744
745std::string LDBCommand::PrintKeyValue(const std::string& key,
746 const std::string& value, bool is_hex) {
747 return PrintKeyValue(key, value, is_hex, is_hex);
748}
749
750std::string LDBCommand::HelpRangeCmdArgs() {
751 std::ostringstream str_stream;
752 str_stream << " ";
753 str_stream << "[--" << ARG_FROM << "] ";
754 str_stream << "[--" << ARG_TO << "] ";
755 return str_stream.str();
756}
757
758bool LDBCommand::IsKeyHex(const std::map<std::string, std::string>& options,
759 const std::vector<std::string>& flags) {
760 return (IsFlagPresent(flags, ARG_HEX) || IsFlagPresent(flags, ARG_KEY_HEX) ||
761 ParseBooleanOption(options, ARG_HEX, false) ||
762 ParseBooleanOption(options, ARG_KEY_HEX, false));
763}
764
765bool LDBCommand::IsValueHex(const std::map<std::string, std::string>& options,
766 const std::vector<std::string>& flags) {
767 return (IsFlagPresent(flags, ARG_HEX) ||
768 IsFlagPresent(flags, ARG_VALUE_HEX) ||
769 ParseBooleanOption(options, ARG_HEX, false) ||
770 ParseBooleanOption(options, ARG_VALUE_HEX, false));
771}
772
773bool LDBCommand::ParseBooleanOption(
774 const std::map<std::string, std::string>& options,
775 const std::string& option, bool default_val) {
776 std::map<std::string, std::string>::const_iterator itr = options.find(option);
777 if (itr != options.end()) {
778 std::string option_val = itr->second;
779 return StringToBool(itr->second);
780 }
781 return default_val;
782}
783
784bool LDBCommand::StringToBool(std::string val) {
785 std::transform(val.begin(), val.end(), val.begin(),
786 [](char ch) -> char { return (char)::tolower(ch); });
787
788 if (val == "true") {
789 return true;
790 } else if (val == "false") {
791 return false;
792 } else {
793 throw "Invalid value for boolean argument";
794 }
795}
796
797CompactorCommand::CompactorCommand(
798 const std::vector<std::string>& /*params*/,
799 const std::map<std::string, std::string>& options,
800 const std::vector<std::string>& flags)
801 : LDBCommand(options, flags, false,
802 BuildCmdLineOptions({ARG_FROM, ARG_TO, ARG_HEX, ARG_KEY_HEX,
803 ARG_VALUE_HEX, ARG_TTL})),
804 null_from_(true),
805 null_to_(true) {
806 std::map<std::string, std::string>::const_iterator itr =
807 options.find(ARG_FROM);
808 if (itr != options.end()) {
809 null_from_ = false;
810 from_ = itr->second;
811 }
812
813 itr = options.find(ARG_TO);
814 if (itr != options.end()) {
815 null_to_ = false;
816 to_ = itr->second;
817 }
818
819 if (is_key_hex_) {
820 if (!null_from_) {
821 from_ = HexToString(from_);
822 }
823 if (!null_to_) {
824 to_ = HexToString(to_);
825 }
826 }
827}
828
829void CompactorCommand::Help(std::string& ret) {
830 ret.append(" ");
831 ret.append(CompactorCommand::Name());
832 ret.append(HelpRangeCmdArgs());
833 ret.append("\n");
834}
835
836void CompactorCommand::DoCommand() {
837 if (!db_) {
838 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
839 return;
840 }
841
842 Slice* begin = nullptr;
843 Slice* end = nullptr;
844 if (!null_from_) {
845 begin = new Slice(from_);
846 }
847 if (!null_to_) {
848 end = new Slice(to_);
849 }
850
851 CompactRangeOptions cro;
852 cro.bottommost_level_compaction = BottommostLevelCompaction::kForce;
853
854 db_->CompactRange(cro, GetCfHandle(), begin, end);
855 exec_state_ = LDBCommandExecuteResult::Succeed("");
856
857 delete begin;
858 delete end;
859}
860
861// ----------------------------------------------------------------------------
862
863const std::string DBLoaderCommand::ARG_DISABLE_WAL = "disable_wal";
864const std::string DBLoaderCommand::ARG_BULK_LOAD = "bulk_load";
865const std::string DBLoaderCommand::ARG_COMPACT = "compact";
866
867DBLoaderCommand::DBLoaderCommand(
868 const std::vector<std::string>& /*params*/,
869 const std::map<std::string, std::string>& options,
870 const std::vector<std::string>& flags)
871 : LDBCommand(
872 options, flags, false,
873 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
874 ARG_TO, ARG_CREATE_IF_MISSING, ARG_DISABLE_WAL,
875 ARG_BULK_LOAD, ARG_COMPACT})),
876 disable_wal_(false),
877 bulk_load_(false),
878 compact_(false) {
879 create_if_missing_ = IsFlagPresent(flags, ARG_CREATE_IF_MISSING);
880 disable_wal_ = IsFlagPresent(flags, ARG_DISABLE_WAL);
881 bulk_load_ = IsFlagPresent(flags, ARG_BULK_LOAD);
882 compact_ = IsFlagPresent(flags, ARG_COMPACT);
883}
884
885void DBLoaderCommand::Help(std::string& ret) {
886 ret.append(" ");
887 ret.append(DBLoaderCommand::Name());
888 ret.append(" [--" + ARG_CREATE_IF_MISSING + "]");
889 ret.append(" [--" + ARG_DISABLE_WAL + "]");
890 ret.append(" [--" + ARG_BULK_LOAD + "]");
891 ret.append(" [--" + ARG_COMPACT + "]");
892 ret.append("\n");
893}
894
895Options DBLoaderCommand::PrepareOptionsForOpenDB() {
896 Options opt = LDBCommand::PrepareOptionsForOpenDB();
897 opt.create_if_missing = create_if_missing_;
898 if (bulk_load_) {
899 opt.PrepareForBulkLoad();
900 }
901 return opt;
902}
903
904void DBLoaderCommand::DoCommand() {
905 if (!db_) {
906 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
907 return;
908 }
909
910 WriteOptions write_options;
911 if (disable_wal_) {
912 write_options.disableWAL = true;
913 }
914
915 int bad_lines = 0;
916 std::string line;
917 // prefer ifstream getline performance vs that from std::cin istream
918 std::ifstream ifs_stdin("/dev/stdin");
919 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
920 while (getline(*istream_p, line, '\n')) {
921 std::string key;
922 std::string value;
923 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
924 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
925 } else if (0 == line.find("Keys in range:")) {
926 // ignore this line
927 } else if (0 == line.find("Created bg thread 0x")) {
928 // ignore this line
929 } else {
930 bad_lines ++;
931 }
932 }
933
934 if (bad_lines > 0) {
935 std::cout << "Warning: " << bad_lines << " bad lines ignored." << std::endl;
936 }
937 if (compact_) {
938 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
939 }
940}
941
942// ----------------------------------------------------------------------------
943
944namespace {
945
946void DumpManifestFile(Options options, std::string file, bool verbose, bool hex,
947 bool json) {
948 EnvOptions sopt;
949 std::string dbname("dummy");
950 std::shared_ptr<Cache> tc(NewLRUCache(options.max_open_files - 10,
951 options.table_cache_numshardbits));
952 // Notice we are using the default options not through SanitizeOptions(),
953 // if VersionSet::DumpManifest() depends on any option done by
954 // SanitizeOptions(), we need to initialize it manually.
955 options.db_paths.emplace_back("dummy", 0);
956 options.num_levels = 64;
957 WriteController wc(options.delayed_write_rate);
958 WriteBufferManager wb(options.db_write_buffer_size);
959 ImmutableDBOptions immutable_db_options(options);
960 VersionSet versions(dbname, &immutable_db_options, sopt, tc.get(), &wb, &wc);
961 Status s = versions.DumpManifest(options, file, verbose, hex, json);
962 if (!s.ok()) {
963 printf("Error in processing file %s %s\n", file.c_str(),
964 s.ToString().c_str());
965 }
966}
967
968} // namespace
969
970const std::string ManifestDumpCommand::ARG_VERBOSE = "verbose";
971const std::string ManifestDumpCommand::ARG_JSON = "json";
972const std::string ManifestDumpCommand::ARG_PATH = "path";
973
974void ManifestDumpCommand::Help(std::string& ret) {
975 ret.append(" ");
976 ret.append(ManifestDumpCommand::Name());
977 ret.append(" [--" + ARG_VERBOSE + "]");
978 ret.append(" [--" + ARG_JSON + "]");
979 ret.append(" [--" + ARG_PATH + "=<path_to_manifest_file>]");
980 ret.append("\n");
981}
982
983ManifestDumpCommand::ManifestDumpCommand(
984 const std::vector<std::string>& /*params*/,
985 const std::map<std::string, std::string>& options,
986 const std::vector<std::string>& flags)
987 : LDBCommand(
988 options, flags, false,
989 BuildCmdLineOptions({ARG_VERBOSE, ARG_PATH, ARG_HEX, ARG_JSON})),
990 verbose_(false),
991 json_(false),
992 path_("") {
993 verbose_ = IsFlagPresent(flags, ARG_VERBOSE);
994 json_ = IsFlagPresent(flags, ARG_JSON);
995
996 std::map<std::string, std::string>::const_iterator itr =
997 options.find(ARG_PATH);
998 if (itr != options.end()) {
999 path_ = itr->second;
1000 if (path_.empty()) {
1001 exec_state_ = LDBCommandExecuteResult::Failed("--path: missing pathname");
1002 }
1003 }
1004}
1005
1006void ManifestDumpCommand::DoCommand() {
1007
1008 std::string manifestfile;
1009
1010 if (!path_.empty()) {
1011 manifestfile = path_;
1012 } else {
1013 bool found = false;
1014 // We need to find the manifest file by searching the directory
1015 // containing the db for files of the form MANIFEST_[0-9]+
1016
1017 auto CloseDir = [](DIR* p) { closedir(p); };
1018 std::unique_ptr<DIR, decltype(CloseDir)> d(opendir(db_path_.c_str()),
1019 CloseDir);
1020
1021 if (d == nullptr) {
1022 exec_state_ =
1023 LDBCommandExecuteResult::Failed(db_path_ + " is not a directory");
1024 return;
1025 }
1026 struct dirent* entry;
1027 while ((entry = readdir(d.get())) != nullptr) {
1028 unsigned int match;
1029 uint64_t num;
1030 if (sscanf(entry->d_name, "MANIFEST-%" PRIu64"l" "u" "%n", &num, &match) &&
1031 match == strlen(entry->d_name)) {
1032 if (!found) {
1033 manifestfile = db_path_ + "/" + std::string(entry->d_name);
1034 found = true;
1035 } else {
1036 exec_state_ = LDBCommandExecuteResult::Failed(
1037 "Multiple MANIFEST files found; use --path to select one");
1038 return;
1039 }
1040 }
1041 }
1042 }
1043
1044 if (verbose_) {
1045 printf("Processing Manifest file %s\n", manifestfile.c_str());
1046 }
1047
1048 DumpManifestFile(options_, manifestfile, verbose_, is_key_hex_, json_);
1049
1050 if (verbose_) {
1051 printf("Processing Manifest file %s done\n", manifestfile.c_str());
1052 }
1053}
1054
1055// ----------------------------------------------------------------------------
1056
1057void ListColumnFamiliesCommand::Help(std::string& ret) {
1058 ret.append(" ");
1059 ret.append(ListColumnFamiliesCommand::Name());
1060 ret.append(" full_path_to_db_directory ");
1061 ret.append("\n");
1062}
1063
1064ListColumnFamiliesCommand::ListColumnFamiliesCommand(
1065 const std::vector<std::string>& params,
1066 const std::map<std::string, std::string>& options,
1067 const std::vector<std::string>& flags)
1068 : LDBCommand(options, flags, false, {}) {
1069 if (params.size() != 1) {
1070 exec_state_ = LDBCommandExecuteResult::Failed(
1071 "dbname must be specified for the list_column_families command");
1072 } else {
1073 dbname_ = params[0];
1074 }
1075}
1076
1077void ListColumnFamiliesCommand::DoCommand() {
1078 std::vector<std::string> column_families;
1079 Status s = DB::ListColumnFamilies(DBOptions(), dbname_, &column_families);
1080 if (!s.ok()) {
1081 printf("Error in processing db %s %s\n", dbname_.c_str(),
1082 s.ToString().c_str());
1083 } else {
1084 printf("Column families in %s: \n{", dbname_.c_str());
1085 bool first = true;
1086 for (auto cf : column_families) {
1087 if (!first) {
1088 printf(", ");
1089 }
1090 first = false;
1091 printf("%s", cf.c_str());
1092 }
1093 printf("}\n");
1094 }
1095}
1096
1097void CreateColumnFamilyCommand::Help(std::string& ret) {
1098 ret.append(" ");
1099 ret.append(CreateColumnFamilyCommand::Name());
1100 ret.append(" --db=<db_path> <new_column_family_name>");
1101 ret.append("\n");
1102}
1103
1104CreateColumnFamilyCommand::CreateColumnFamilyCommand(
1105 const std::vector<std::string>& params,
1106 const std::map<std::string, std::string>& options,
1107 const std::vector<std::string>& flags)
1108 : LDBCommand(options, flags, true, {ARG_DB}) {
1109 if (params.size() != 1) {
1110 exec_state_ = LDBCommandExecuteResult::Failed(
1111 "new column family name must be specified");
1112 } else {
1113 new_cf_name_ = params[0];
1114 }
1115}
1116
1117void CreateColumnFamilyCommand::DoCommand() {
1118 ColumnFamilyHandle* new_cf_handle = nullptr;
1119 Status st = db_->CreateColumnFamily(options_, new_cf_name_, &new_cf_handle);
1120 if (st.ok()) {
1121 fprintf(stdoutstdout, "OK\n");
1122 } else {
1123 exec_state_ = LDBCommandExecuteResult::Failed(
1124 "Fail to create new column family: " + st.ToString());
1125 }
1126 delete new_cf_handle;
1127 CloseDB();
1128}
1129
1130// ----------------------------------------------------------------------------
1131
1132namespace {
1133
1134std::string ReadableTime(int unixtime) {
1135 char time_buffer [80];
1136 time_t rawtime = unixtime;
1137 struct tm tInfo;
1138 struct tm* timeinfo = localtime_r(&rawtime, &tInfo);
1139 assert(timeinfo == &tInfo)(static_cast<void> (0));
1140 strftime(time_buffer, 80, "%c", timeinfo);
1141 return std::string(time_buffer);
1142}
1143
1144// This function only called when it's the sane case of >1 buckets in time-range
1145// Also called only when timekv falls between ttl_start and ttl_end provided
1146void IncBucketCounts(std::vector<uint64_t>& bucket_counts, int ttl_start,
1147 int time_range, int bucket_size, int timekv,
1148 int num_buckets) {
1149#ifdef NDEBUG1
1150 (void)time_range;
1151 (void)num_buckets;
1152#endif
1153 assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&(static_cast<void> (0))
1154 timekv < (ttl_start + time_range) && num_buckets > 1)(static_cast<void> (0));
1155 int bucket = (timekv - ttl_start) / bucket_size;
1156 bucket_counts[bucket]++;
1157}
1158
1159void PrintBucketCounts(const std::vector<uint64_t>& bucket_counts,
1160 int ttl_start, int ttl_end, int bucket_size,
1161 int num_buckets) {
1162 int time_point = ttl_start;
1163 for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
1164 fprintf(stdoutstdout, "Keys in range %s to %s : %lu\n",
1165 ReadableTime(time_point).c_str(),
1166 ReadableTime(time_point + bucket_size).c_str(),
1167 (unsigned long)bucket_counts[i]);
1168 }
1169 fprintf(stdoutstdout, "Keys in range %s to %s : %lu\n",
1170 ReadableTime(time_point).c_str(),
1171 ReadableTime(ttl_end).c_str(),
1172 (unsigned long)bucket_counts[num_buckets - 1]);
1173}
1174
1175} // namespace
1176
1177const std::string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
1178const std::string InternalDumpCommand::ARG_COUNT_DELIM = "count_delim";
1179const std::string InternalDumpCommand::ARG_STATS = "stats";
1180const std::string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";
1181
1182InternalDumpCommand::InternalDumpCommand(
1183 const std::vector<std::string>& /*params*/,
1184 const std::map<std::string, std::string>& options,
1185 const std::vector<std::string>& flags)
1186 : LDBCommand(
1187 options, flags, true,
1188 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1189 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY,
1190 ARG_COUNT_DELIM, ARG_STATS, ARG_INPUT_KEY_HEX})),
1191 has_from_(false),
1192 has_to_(false),
1193 max_keys_(-1),
1194 delim_("."),
1195 count_only_(false),
1196 count_delim_(false),
1197 print_stats_(false),
1198 is_input_key_hex_(false) {
1199 has_from_ = ParseStringOption(options, ARG_FROM, &from_);
1200 has_to_ = ParseStringOption(options, ARG_TO, &to_);
1201
1202 ParseIntOption(options, ARG_MAX_KEYS, max_keys_, exec_state_);
1203 std::map<std::string, std::string>::const_iterator itr =
1204 options.find(ARG_COUNT_DELIM);
1205 if (itr != options.end()) {
1206 delim_ = itr->second;
1207 count_delim_ = true;
1208 // fprintf(stdout,"delim = %c\n",delim_[0]);
1209 } else {
1210 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1211 delim_=".";
1212 }
1213
1214 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1215 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1216 is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);
1217
1218 if (is_input_key_hex_) {
1219 if (has_from_) {
1220 from_ = HexToString(from_);
1221 }
1222 if (has_to_) {
1223 to_ = HexToString(to_);
1224 }
1225 }
1226}
1227
1228void InternalDumpCommand::Help(std::string& ret) {
1229 ret.append(" ");
1230 ret.append(InternalDumpCommand::Name());
1231 ret.append(HelpRangeCmdArgs());
1232 ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
1233 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1234 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1235 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1236 ret.append(" [--" + ARG_STATS + "]");
1237 ret.append("\n");
1238}
1239
1240void InternalDumpCommand::DoCommand() {
1241 if (!db_) {
1242 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
1243 return;
1244 }
1245
1246 if (print_stats_) {
1247 std::string stats;
1248 if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1249 fprintf(stdoutstdout, "%s\n", stats.c_str());
1250 }
1251 }
1252
1253 // Cast as DBImpl to get internal iterator
1254 std::vector<KeyVersion> key_versions;
1255 Status st = GetAllKeyVersions(db_, from_, to_, max_keys_, &key_versions);
1256 if (!st.ok()) {
1257 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1258 return;
1259 }
1260 std::string rtype1, rtype2, row, val;
1261 rtype2 = "";
1262 uint64_t c=0;
1263 uint64_t s1=0,s2=0;
1264
1265 long long count = 0;
1266 for (auto& key_version : key_versions) {
1267 InternalKey ikey(key_version.user_key, key_version.sequence,
1268 static_cast<ValueType>(key_version.type));
1269 if (has_to_ && ikey.user_key() == to_) {
1270 // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1271 // traditionally excluded such keys.
1272 break;
1273 }
1274 ++count;
1275 int k;
1276 if (count_delim_) {
1277 rtype1 = "";
1278 s1=0;
1279 row = ikey.Encode().ToString();
1280 val = key_version.value;
1281 for(k=0;row[k]!='\x01' && row[k]!='\0';k++)
1282 s1++;
1283 for(k=0;val[k]!='\x01' && val[k]!='\0';k++)
1284 s1++;
1285 for(int j=0;row[j]!=delim_[0] && row[j]!='\0' && row[j]!='\x01';j++)
1286 rtype1+=row[j];
1287 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1288 fprintf(stdoutstdout, "%s => count:%" PRIu64"l" "u" "\tsize:%" PRIu64"l" "u" "\n",
1289 rtype2.c_str(), c, s2);
1290 c=1;
1291 s2=s1;
1292 rtype2 = rtype1;
1293 } else {
1294 c++;
1295 s2+=s1;
1296 rtype2=rtype1;
1297 }
1298 }
1299
1300 if (!count_only_ && !count_delim_) {
1301 std::string key = ikey.DebugString(is_key_hex_);
1302 std::string value = Slice(key_version.value).ToString(is_value_hex_);
1303 std::cout << key << " => " << value << "\n";
1304 }
1305
1306 // Terminate if maximum number of keys have been dumped
1307 if (max_keys_ > 0 && count >= max_keys_) break;
1308 }
1309 if(count_delim_) {
1310 fprintf(stdoutstdout, "%s => count:%" PRIu64"l" "u" "\tsize:%" PRIu64"l" "u" "\n",
1311 rtype2.c_str(), c, s2);
1312 } else {
1313 fprintf(stdoutstdout, "Internal keys in range: %lld\n", count);
1314 }
1315}
1316
1317const std::string DBDumperCommand::ARG_COUNT_ONLY = "count_only";
1318const std::string DBDumperCommand::ARG_COUNT_DELIM = "count_delim";
1319const std::string DBDumperCommand::ARG_STATS = "stats";
1320const std::string DBDumperCommand::ARG_TTL_BUCKET = "bucket";
1321
1322DBDumperCommand::DBDumperCommand(
1323 const std::vector<std::string>& /*params*/,
1324 const std::map<std::string, std::string>& options,
1325 const std::vector<std::string>& flags)
1326 : LDBCommand(options, flags, true,
1327 BuildCmdLineOptions(
1328 {ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
1329 ARG_TO, ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_COUNT_DELIM,
1330 ARG_STATS, ARG_TTL_START, ARG_TTL_END, ARG_TTL_BUCKET,
1331 ARG_TIMESTAMP, ARG_PATH})),
1332 null_from_(true),
1333 null_to_(true),
1334 max_keys_(-1),
1335 count_only_(false),
1336 count_delim_(false),
1337 print_stats_(false) {
1338 std::map<std::string, std::string>::const_iterator itr =
1339 options.find(ARG_FROM);
1340 if (itr != options.end()) {
1341 null_from_ = false;
1342 from_ = itr->second;
1343 }
1344
1345 itr = options.find(ARG_TO);
1346 if (itr != options.end()) {
1347 null_to_ = false;
1348 to_ = itr->second;
1349 }
1350
1351 itr = options.find(ARG_MAX_KEYS);
1352 if (itr != options.end()) {
1353 try {
1354#if defined(CYGWIN)
1355 max_keys_ = strtol(itr->second.c_str(), 0, 10);
1356#else
1357 max_keys_ = std::stoi(itr->second);
1358#endif
1359 } catch (const std::invalid_argument&) {
1360 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
1361 " has an invalid value");
1362 } catch (const std::out_of_range&) {
1363 exec_state_ = LDBCommandExecuteResult::Failed(
1364 ARG_MAX_KEYS + " has a value out-of-range");
1365 }
1366 }
1367 itr = options.find(ARG_COUNT_DELIM);
1368 if (itr != options.end()) {
1369 delim_ = itr->second;
1370 count_delim_ = true;
1371 } else {
1372 count_delim_ = IsFlagPresent(flags, ARG_COUNT_DELIM);
1373 delim_=".";
1374 }
1375
1376 print_stats_ = IsFlagPresent(flags, ARG_STATS);
1377 count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
1378
1379 if (is_key_hex_) {
1380 if (!null_from_) {
1381 from_ = HexToString(from_);
1382 }
1383 if (!null_to_) {
1384 to_ = HexToString(to_);
1385 }
1386 }
1387
1388 itr = options.find(ARG_PATH);
1389 if (itr != options.end()) {
1390 path_ = itr->second;
1391 if (db_path_.empty()) {
1392 db_path_ = path_;
1393 }
1394 }
1395}
1396
1397void DBDumperCommand::Help(std::string& ret) {
1398 ret.append(" ");
1399 ret.append(DBDumperCommand::Name());
1400 ret.append(HelpRangeCmdArgs());
1401 ret.append(" [--" + ARG_TTL + "]");
1402 ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
1403 ret.append(" [--" + ARG_TIMESTAMP + "]");
1404 ret.append(" [--" + ARG_COUNT_ONLY + "]");
1405 ret.append(" [--" + ARG_COUNT_DELIM + "=<char>]");
1406 ret.append(" [--" + ARG_STATS + "]");
1407 ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
1408 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
1409 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
1410 ret.append(" [--" + ARG_PATH + "=<path_to_a_file>]");
1411 ret.append("\n");
1412}
1413
1414/**
1415 * Handles two separate cases:
1416 *
1417 * 1) --db is specified - just dump the database.
1418 *
1419 * 2) --path is specified - determine based on file extension what dumping
1420 * function to call. Please note that we intentionally use the extension
1421 * and avoid probing the file contents under the assumption that renaming
1422 * the files is not a supported scenario.
1423 *
1424 */
1425void DBDumperCommand::DoCommand() {
1426 if (!db_) {
1427 assert(!path_.empty())(static_cast<void> (0));
1428 std::string fileName = GetFileNameFromPath(path_);
1429 uint64_t number;
1430 FileType type;
1431
1432 exec_state_ = LDBCommandExecuteResult::Succeed("");
1433
1434 if (!ParseFileName(fileName, &number, &type)) {
1435 exec_state_ =
1436 LDBCommandExecuteResult::Failed("Can't parse file type: " + path_);
1437 return;
1438 }
1439
1440 switch (type) {
1441 case kLogFile:
1442 // TODO(myabandeh): allow configuring is_write_commited
1443 DumpWalFile(options_, path_, /* print_header_ */ true,
1444 /* print_values_ */ true, true /* is_write_commited */,
1445 &exec_state_);
1446 break;
1447 case kTableFile:
1448 DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true);
1449 break;
1450 case kDescriptorFile:
1451 DumpManifestFile(options_, path_, /* verbose_ */ false, is_key_hex_,
1452 /* json_ */ false);
1453 break;
1454 default:
1455 exec_state_ = LDBCommandExecuteResult::Failed(
1456 "File type not supported: " + path_);
1457 break;
1458 }
1459
1460 } else {
1461 DoDumpCommand();
1462 }
1463}
1464
1465void DBDumperCommand::DoDumpCommand() {
1466 assert(nullptr != db_)(static_cast<void> (0));
1467 assert(path_.empty())(static_cast<void> (0));
1468
1469 // Parse command line args
1470 uint64_t count = 0;
1471 if (print_stats_) {
1472 std::string stats;
1473 if (db_->GetProperty("rocksdb.stats", &stats)) {
1474 fprintf(stdoutstdout, "%s\n", stats.c_str());
1475 }
1476 }
1477
1478 // Setup key iterator
1479 ReadOptions scan_read_opts;
1480 scan_read_opts.total_order_seek = true;
1481 Iterator* iter = db_->NewIterator(scan_read_opts, GetCfHandle());
1482 Status st = iter->status();
1483 if (!st.ok()) {
1484 exec_state_ =
1485 LDBCommandExecuteResult::Failed("Iterator error." + st.ToString());
1486 }
1487
1488 if (!null_from_) {
1489 iter->Seek(from_);
1490 } else {
1491 iter->SeekToFirst();
1492 }
1493
1494 int max_keys = max_keys_;
1495 int ttl_start;
1496 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
1497 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
1498 }
1499 int ttl_end;
1500 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
1501 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
1502 }
1503 if (ttl_end < ttl_start) {
1504 fprintf(stderrstderr, "Error: End time can't be less than start time\n");
1505 delete iter;
1506 return;
1507 }
1508 int time_range = ttl_end - ttl_start;
1509 int bucket_size;
1510 if (!ParseIntOption(option_map_, ARG_TTL_BUCKET, bucket_size, exec_state_) ||
1511 bucket_size <= 0) {
1512 bucket_size = time_range; // Will have just 1 bucket by default
1513 }
1514 //cretaing variables for row count of each type
1515 std::string rtype1, rtype2, row, val;
1516 rtype2 = "";
1517 uint64_t c=0;
1518 uint64_t s1=0,s2=0;
1519
1520 // At this point, bucket_size=0 => time_range=0
1521 int num_buckets = (bucket_size >= time_range)
1522 ? 1
1523 : ((time_range + bucket_size - 1) / bucket_size);
1524 std::vector<uint64_t> bucket_counts(num_buckets, 0);
1525 if (is_db_ttl_ && !count_only_ && timestamp_ && !count_delim_) {
1526 fprintf(stdoutstdout, "Dumping key-values from %s to %s\n",
1527 ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
1528 }
1529
1530 HistogramImpl vsize_hist;
1531
1532 for (; iter->Valid(); iter->Next()) {
1533 int rawtime = 0;
1534 // If end marker was specified, we stop before it
1535 if (!null_to_ && (iter->key().ToString() >= to_))
1536 break;
1537 // Terminate if maximum number of keys have been dumped
1538 if (max_keys == 0)
1539 break;
1540 if (is_db_ttl_) {
1541 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(iter);
1542 rawtime = it_ttl->timestamp();
1543 if (rawtime < ttl_start || rawtime >= ttl_end) {
1544 continue;
1545 }
1546 }
1547 if (max_keys > 0) {
1548 --max_keys;
1549 }
1550 if (is_db_ttl_ && num_buckets > 1) {
1551 IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
1552 rawtime, num_buckets);
1553 }
1554 ++count;
1555 if (count_delim_) {
1556 rtype1 = "";
1557 row = iter->key().ToString();
1558 val = iter->value().ToString();
1559 s1 = row.size()+val.size();
1560 for(int j=0;row[j]!=delim_[0] && row[j]!='\0';j++)
1561 rtype1+=row[j];
1562 if(rtype2.compare("") && rtype2.compare(rtype1)!=0) {
1563 fprintf(stdoutstdout, "%s => count:%" PRIu64"l" "u" "\tsize:%" PRIu64"l" "u" "\n",
1564 rtype2.c_str(), c, s2);
1565 c=1;
1566 s2=s1;
1567 rtype2 = rtype1;
1568 } else {
1569 c++;
1570 s2+=s1;
1571 rtype2=rtype1;
1572 }
1573
1574 }
1575
1576 if (count_only_) {
1577 vsize_hist.Add(iter->value().size());
1578 }
1579
1580 if (!count_only_ && !count_delim_) {
1581 if (is_db_ttl_ && timestamp_) {
1582 fprintf(stdoutstdout, "%s ", ReadableTime(rawtime).c_str());
1583 }
1584 std::string str =
1585 PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
1586 is_key_hex_, is_value_hex_);
1587 fprintf(stdoutstdout, "%s\n", str.c_str());
1588 }
1589 }
1590
1591 if (num_buckets > 1 && is_db_ttl_) {
1592 PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
1593 num_buckets);
1594 } else if(count_delim_) {
1595 fprintf(stdoutstdout, "%s => count:%" PRIu64"l" "u" "\tsize:%" PRIu64"l" "u" "\n",
1596 rtype2.c_str(), c, s2);
1597 } else {
1598 fprintf(stdoutstdout, "Keys in range: %" PRIu64"l" "u" "\n", count);
1599 }
1600
1601 if (count_only_) {
1602 fprintf(stdoutstdout, "Value size distribution: \n");
1603 fprintf(stdoutstdout, "%s\n", vsize_hist.ToString().c_str());
1604 }
1605 // Clean up
1606 delete iter;
1607}
1608
1609const std::string ReduceDBLevelsCommand::ARG_NEW_LEVELS = "new_levels";
1610const std::string ReduceDBLevelsCommand::ARG_PRINT_OLD_LEVELS =
1611 "print_old_levels";
1612
1613ReduceDBLevelsCommand::ReduceDBLevelsCommand(
1614 const std::vector<std::string>& /*params*/,
1615 const std::map<std::string, std::string>& options,
1616 const std::vector<std::string>& flags)
1617 : LDBCommand(options, flags, false,
1618 BuildCmdLineOptions({ARG_NEW_LEVELS, ARG_PRINT_OLD_LEVELS})),
1619 old_levels_(1 << 7),
1620 new_levels_(-1),
1621 print_old_levels_(false) {
1622 ParseIntOption(option_map_, ARG_NEW_LEVELS, new_levels_, exec_state_);
1623 print_old_levels_ = IsFlagPresent(flags, ARG_PRINT_OLD_LEVELS);
1624
1625 if(new_levels_ <= 0) {
1626 exec_state_ = LDBCommandExecuteResult::Failed(
1627 " Use --" + ARG_NEW_LEVELS + " to specify a new level number\n");
1628 }
1629}
1630
1631std::vector<std::string> ReduceDBLevelsCommand::PrepareArgs(
1632 const std::string& db_path, int new_levels, bool print_old_level) {
1633 std::vector<std::string> ret;
1634 ret.push_back("reduce_levels");
1635 ret.push_back("--" + ARG_DB + "=" + db_path);
1636 ret.push_back("--" + ARG_NEW_LEVELS + "=" + rocksdb::ToString(new_levels));
1637 if(print_old_level) {
1638 ret.push_back("--" + ARG_PRINT_OLD_LEVELS);
1639 }
1640 return ret;
1641}
1642
1643void ReduceDBLevelsCommand::Help(std::string& ret) {
1644 ret.append(" ");
1645 ret.append(ReduceDBLevelsCommand::Name());
1646 ret.append(" --" + ARG_NEW_LEVELS + "=<New number of levels>");
1647 ret.append(" [--" + ARG_PRINT_OLD_LEVELS + "]");
1648 ret.append("\n");
1649}
1650
1651Options ReduceDBLevelsCommand::PrepareOptionsForOpenDB() {
1652 Options opt = LDBCommand::PrepareOptionsForOpenDB();
1653 opt.num_levels = old_levels_;
1654 opt.max_bytes_for_level_multiplier_additional.resize(opt.num_levels, 1);
1655 // Disable size compaction
1656 opt.max_bytes_for_level_base = 1ULL << 50;
1657 opt.max_bytes_for_level_multiplier = 1;
1658 return opt;
1659}
1660
1661Status ReduceDBLevelsCommand::GetOldNumOfLevels(Options& opt,
1662 int* levels) {
1663 ImmutableDBOptions db_options(opt);
1664 EnvOptions soptions;
1665 std::shared_ptr<Cache> tc(
1666 NewLRUCache(opt.max_open_files - 10, opt.table_cache_numshardbits));
1667 const InternalKeyComparator cmp(opt.comparator);
1668 WriteController wc(opt.delayed_write_rate);
1669 WriteBufferManager wb(opt.db_write_buffer_size);
1670 VersionSet versions(db_path_, &db_options, soptions, tc.get(), &wb, &wc);
1671 std::vector<ColumnFamilyDescriptor> dummy;
1672 ColumnFamilyDescriptor dummy_descriptor(kDefaultColumnFamilyName,
1673 ColumnFamilyOptions(opt));
1674 dummy.push_back(dummy_descriptor);
1675 // We rely the VersionSet::Recover to tell us the internal data structures
1676 // in the db. And the Recover() should never do any change
1677 // (like LogAndApply) to the manifest file.
1678 Status st = versions.Recover(dummy);
1679 if (!st.ok()) {
1680 return st;
1681 }
1682 int max = -1;
1683 auto default_cfd = versions.GetColumnFamilySet()->GetDefault();
1684 for (int i = 0; i < default_cfd->NumberLevels(); i++) {
1685 if (default_cfd->current()->storage_info()->NumLevelFiles(i)) {
1686 max = i;
1687 }
1688 }
1689
1690 *levels = max + 1;
1691 return st;
1692}
1693
1694void ReduceDBLevelsCommand::DoCommand() {
1695 if (new_levels_ <= 1) {
1
Assuming the condition is false
→
2
←
Taking false branch
→
1696 exec_state_ =
1697 LDBCommandExecuteResult::Failed("Invalid number of levels.\n");
1698 return;
1699 }
1700
1701 Status st;
1702 Options opt = PrepareOptionsForOpenDB();
1703 int old_level_num = -1;
1704 st = GetOldNumOfLevels(opt, &old_level_num);
1705 if (!st.ok()) {
3
←
Taking false branch
→
1706 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1707 return;
1708 }
1709
1710 if (print_old_levels_) {
4
←
Assuming the condition is false
→
5
←
Taking false branch
→
1711 fprintf(stdoutstdout, "The old number of levels in use is %d\n", old_level_num);
1712 }
1713
1714 if (old_level_num <= new_levels_) {
6
←
Taking false branch
→
1715 return;
1716 }
1717
1718 old_levels_ = old_level_num;
1719
1720 OpenDB();
7
←
Calling 'LDBCommand::OpenDB'
→
13
←
Returning from 'LDBCommand::OpenDB'
→
1721 if (exec_state_.IsFailed()) {
14
←
Taking false branch
→
1722 return;
1723 }
1724 assert(db_ != nullptr)(static_cast<void> (0));
1725 // Compact the whole DB to put all files to the highest level.
1726 fprintf(stdoutstdout, "Compacting the db...\n");
1727 db_->CompactRange(CompactRangeOptions(), GetCfHandle(), nullptr, nullptr);
15
←
Calling 'LDBCommand::GetCfHandle'
→
1728 CloseDB();
1729
1730 EnvOptions soptions;
1731 st = VersionSet::ReduceNumberOfLevels(db_path_, &opt, soptions, new_levels_);
1732 if (!st.ok()) {
1733 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
1734 return;
1735 }
1736}
1737
1738const std::string ChangeCompactionStyleCommand::ARG_OLD_COMPACTION_STYLE =
1739 "old_compaction_style";
1740const std::string ChangeCompactionStyleCommand::ARG_NEW_COMPACTION_STYLE =
1741 "new_compaction_style";
1742
1743ChangeCompactionStyleCommand::ChangeCompactionStyleCommand(
1744 const std::vector<std::string>& /*params*/,
1745 const std::map<std::string, std::string>& options,
1746 const std::vector<std::string>& flags)
1747 : LDBCommand(options, flags, false,
1748 BuildCmdLineOptions(
1749 {ARG_OLD_COMPACTION_STYLE, ARG_NEW_COMPACTION_STYLE})),
1750 old_compaction_style_(-1),
1751 new_compaction_style_(-1) {
1752 ParseIntOption(option_map_, ARG_OLD_COMPACTION_STYLE, old_compaction_style_,
1753 exec_state_);
1754 if (old_compaction_style_ != kCompactionStyleLevel &&
1755 old_compaction_style_ != kCompactionStyleUniversal) {
1756 exec_state_ = LDBCommandExecuteResult::Failed(
1757 "Use --" + ARG_OLD_COMPACTION_STYLE + " to specify old compaction " +
1758 "style. Check ldb help for proper compaction style value.\n");
1759 return;
1760 }
1761
1762 ParseIntOption(option_map_, ARG_NEW_COMPACTION_STYLE, new_compaction_style_,
1763 exec_state_);
1764 if (new_compaction_style_ != kCompactionStyleLevel &&
1765 new_compaction_style_ != kCompactionStyleUniversal) {
1766 exec_state_ = LDBCommandExecuteResult::Failed(
1767 "Use --" + ARG_NEW_COMPACTION_STYLE + " to specify new compaction " +
1768 "style. Check ldb help for proper compaction style value.\n");
1769 return;
1770 }
1771
1772 if (new_compaction_style_ == old_compaction_style_) {
1773 exec_state_ = LDBCommandExecuteResult::Failed(
1774 "Old compaction style is the same as new compaction style. "
1775 "Nothing to do.\n");
1776 return;
1777 }
1778
1779 if (old_compaction_style_ == kCompactionStyleUniversal &&
1780 new_compaction_style_ == kCompactionStyleLevel) {
1781 exec_state_ = LDBCommandExecuteResult::Failed(
1782 "Convert from universal compaction to level compaction. "
1783 "Nothing to do.\n");
1784 return;
1785 }
1786}
1787
1788void ChangeCompactionStyleCommand::Help(std::string& ret) {
1789 ret.append(" ");
1790 ret.append(ChangeCompactionStyleCommand::Name());
1791 ret.append(" --" + ARG_OLD_COMPACTION_STYLE + "=<Old compaction style: 0 " +
1792 "for level compaction, 1 for universal compaction>");
1793 ret.append(" --" + ARG_NEW_COMPACTION_STYLE + "=<New compaction style: 0 " +
1794 "for level compaction, 1 for universal compaction>");
1795 ret.append("\n");
1796}
1797
1798Options ChangeCompactionStyleCommand::PrepareOptionsForOpenDB() {
1799 Options opt = LDBCommand::PrepareOptionsForOpenDB();
1800
1801 if (old_compaction_style_ == kCompactionStyleLevel &&
1802 new_compaction_style_ == kCompactionStyleUniversal) {
1803 // In order to convert from level compaction to universal compaction, we
1804 // need to compact all data into a single file and move it to level 0.
1805 opt.disable_auto_compactions = true;
1806 opt.target_file_size_base = INT_MAX2147483647;
1807 opt.target_file_size_multiplier = 1;
1808 opt.max_bytes_for_level_base = INT_MAX2147483647;
1809 opt.max_bytes_for_level_multiplier = 1;
1810 }
1811
1812 return opt;
1813}
1814
1815void ChangeCompactionStyleCommand::DoCommand() {
1816 // print db stats before we have made any change
1817 std::string property;
1818 std::string files_per_level;
1819 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1820 db_->GetProperty(GetCfHandle(),
1821 "rocksdb.num-files-at-level" + NumberToString(i),
1822 &property);
1823
1824 // format print string
1825 char buf[100];
1826 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1827 files_per_level += buf;
1828 }
1829 fprintf(stdoutstdout, "files per level before compaction: %s\n",
1830 files_per_level.c_str());
1831
1832 // manual compact into a single file and move the file to level 0
1833 CompactRangeOptions compact_options;
1834 compact_options.change_level = true;
1835 compact_options.target_level = 0;
1836 db_->CompactRange(compact_options, GetCfHandle(), nullptr, nullptr);
1837
1838 // verify compaction result
1839 files_per_level = "";
1840 int num_files = 0;
1841 for (int i = 0; i < db_->NumberLevels(GetCfHandle()); i++) {
1842 db_->GetProperty(GetCfHandle(),
1843 "rocksdb.num-files-at-level" + NumberToString(i),
1844 &property);
1845
1846 // format print string
1847 char buf[100];
1848 snprintf(buf, sizeof(buf), "%s%s", (i ? "," : ""), property.c_str());
1849 files_per_level += buf;
1850
1851 num_files = atoi(property.c_str());
1852
1853 // level 0 should have only 1 file
1854 if (i == 0 && num_files != 1) {
1855 exec_state_ = LDBCommandExecuteResult::Failed(
1856 "Number of db files at "
1857 "level 0 after compaction is " +
1858 ToString(num_files) + ", not 1.\n");
1859 return;
1860 }
1861 // other levels should have no file
1862 if (i > 0 && num_files != 0) {
1863 exec_state_ = LDBCommandExecuteResult::Failed(
1864 "Number of db files at "
1865 "level " +
1866 ToString(i) + " after compaction is " + ToString(num_files) +
1867 ", not 0.\n");
1868 return;
1869 }
1870 }
1871
1872 fprintf(stdoutstdout, "files per level after compaction: %s\n",
1873 files_per_level.c_str());
1874}
1875
1876// ----------------------------------------------------------------------------
1877
1878namespace {
1879
1880struct StdErrReporter : public log::Reader::Reporter {
1881 void Corruption(size_t /*bytes*/, const Status& s) override {
1882 std::cerr << "Corruption detected in log file " << s.ToString() << "\n";
1883 }
1884};
1885
1886class InMemoryHandler : public WriteBatch::Handler {
1887 public:
1888 InMemoryHandler(std::stringstream& row, bool print_values,
1889 bool write_after_commit = false)
1890 : Handler(),
1891 row_(row),
1892 print_values_(print_values),
1893 write_after_commit_(write_after_commit) {}
1894
1895 void commonPutMerge(const Slice& key, const Slice& value) {
1896 std::string k = LDBCommand::StringToHex(key.ToString());
1897 if (print_values_) {
1898 std::string v = LDBCommand::StringToHex(value.ToString());
1899 row_ << k << " : ";
1900 row_ << v << " ";
1901 } else {
1902 row_ << k << " ";
1903 }
1904 }
1905
1906 Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
1907 row_ << "PUT(" << cf << ") : ";
1908 commonPutMerge(key, value);
1909 return Status::OK();
1910 }
1911
1912 Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
1913 row_ << "MERGE(" << cf << ") : ";
1914 commonPutMerge(key, value);
1915 return Status::OK();
1916 }
1917
1918 Status MarkNoop(bool) override {
1919 row_ << "NOOP ";
1920 return Status::OK();
1921 }
1922
1923 Status DeleteCF(uint32_t cf, const Slice& key) override {
1924 row_ << "DELETE(" << cf << ") : ";
1925 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1926 return Status::OK();
1927 }
1928
1929 Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
1930 row_ << "SINGLE_DELETE(" << cf << ") : ";
1931 row_ << LDBCommand::StringToHex(key.ToString()) << " ";
1932 return Status::OK();
1933 }
1934
1935 Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
1936 const Slice& end_key) override {
1937 row_ << "DELETE_RANGE(" << cf << ") : ";
1938 row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
1939 row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
1940 return Status::OK();
1941 }
1942
1943 Status MarkBeginPrepare(bool unprepare) override {
1944 row_ << "BEGIN_PREPARE(";
1945 row_ << (unprepare ? "true" : "false") << ") ";
1946 return Status::OK();
1947 }
1948
1949 Status MarkEndPrepare(const Slice& xid) override {
1950 row_ << "END_PREPARE(";
1951 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1952 return Status::OK();
1953 }
1954
1955 Status MarkRollback(const Slice& xid) override {
1956 row_ << "ROLLBACK(";
1957 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1958 return Status::OK();
1959 }
1960
1961 Status MarkCommit(const Slice& xid) override {
1962 row_ << "COMMIT(";
1963 row_ << LDBCommand::StringToHex(xid.ToString()) << ") ";
1964 return Status::OK();
1965 }
1966
1967 ~InMemoryHandler() override {}
1968
1969 protected:
1970 bool WriteAfterCommit() const override { return write_after_commit_; }
1971
1972 private:
1973 std::stringstream& row_;
1974 bool print_values_;
1975 bool write_after_commit_;
1976};
1977
1978void DumpWalFile(Options options, std::string wal_file, bool print_header,
1979 bool print_values, bool is_write_committed,
1980 LDBCommandExecuteResult* exec_state) {
1981 Env* env = options.env;
1982 EnvOptions soptions(options);
1983 std::unique_ptr<SequentialFileReader> wal_file_reader;
1984
1985 Status status;
1986 {
1987 std::unique_ptr<SequentialFile> file;
1988 status = env->NewSequentialFile(wal_file, &file, soptions);
1989 if (status.ok()) {
1990 wal_file_reader.reset(
1991 new SequentialFileReader(std::move(file), wal_file));
1992 }
1993 }
1994 if (!status.ok()) {
1995 if (exec_state) {
1996 *exec_state = LDBCommandExecuteResult::Failed("Failed to open WAL file " +
1997 status.ToString());
1998 } else {
1999 std::cerr << "Error: Failed to open WAL file " << status.ToString()
2000 << std::endl;
2001 }
2002 } else {
2003 StdErrReporter reporter;
2004 uint64_t log_number;
2005 FileType type;
2006
2007 // we need the log number, but ParseFilename expects dbname/NNN.log.
2008 std::string sanitized = wal_file;
2009 size_t lastslash = sanitized.rfind('/');
2010 if (lastslash != std::string::npos)
2011 sanitized = sanitized.substr(lastslash + 1);
2012 if (!ParseFileName(sanitized, &log_number, &type)) {
2013 // bogus input, carry on as best we can
2014 log_number = 0;
2015 }
2016 log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
2017 true /* checksum */, log_number);
2018 std::string scratch;
2019 WriteBatch batch;
2020 Slice record;
2021 std::stringstream row;
2022 if (print_header) {
2023 std::cout << "Sequence,Count,ByteSize,Physical Offset,Key(s)";
2024 if (print_values) {
2025 std::cout << " : value ";
2026 }
2027 std::cout << "\n";
2028 }
2029 while (reader.ReadRecord(&record, &scratch)) {
2030 row.str("");
2031 if (record.size() < WriteBatchInternal::kHeader) {
2032 reporter.Corruption(record.size(),
2033 Status::Corruption("log record too small"));
2034 } else {
2035 WriteBatchInternal::SetContents(&batch, record);
2036 row << WriteBatchInternal::Sequence(&batch) << ",";
2037 row << WriteBatchInternal::Count(&batch) << ",";
2038 row << WriteBatchInternal::ByteSize(&batch) << ",";
2039 row << reader.LastRecordOffset() << ",";
2040 InMemoryHandler handler(row, print_values, is_write_committed);
2041 batch.Iterate(&handler);
2042 row << "\n";
2043 }
2044 std::cout << row.str();
2045 }
2046 }
2047}
2048
2049} // namespace
2050
2051const std::string WALDumperCommand::ARG_WAL_FILE = "walfile";
2052const std::string WALDumperCommand::ARG_WRITE_COMMITTED = "write_committed";
2053const std::string WALDumperCommand::ARG_PRINT_VALUE = "print_value";
2054const std::string WALDumperCommand::ARG_PRINT_HEADER = "header";
2055
2056WALDumperCommand::WALDumperCommand(
2057 const std::vector<std::string>& /*params*/,
2058 const std::map<std::string, std::string>& options,
2059 const std::vector<std::string>& flags)
2060 : LDBCommand(options, flags, true,
2061 BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED,
2062 ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
2063 print_header_(false),
2064 print_values_(false),
2065 is_write_committed_(false) {
2066 wal_file_.clear();
2067
2068 std::map<std::string, std::string>::const_iterator itr =
2069 options.find(ARG_WAL_FILE);
2070 if (itr != options.end()) {
2071 wal_file_ = itr->second;
2072 }
2073
2074
2075 print_header_ = IsFlagPresent(flags, ARG_PRINT_HEADER);
2076 print_values_ = IsFlagPresent(flags, ARG_PRINT_VALUE);
2077 is_write_committed_ = ParseBooleanOption(options, ARG_WRITE_COMMITTED, true);
2078
2079 if (wal_file_.empty()) {
2080 exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2081 " must be specified.");
2082 }
2083}
2084
2085void WALDumperCommand::Help(std::string& ret) {
2086 ret.append(" ");
2087 ret.append(WALDumperCommand::Name());
2088 ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
2089 ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2090 ret.append(" [--" + ARG_PRINT_VALUE + "] ");
2091 ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] ");
2092 ret.append("\n");
2093}
2094
2095void WALDumperCommand::DoCommand() {
2096 DumpWalFile(options_, wal_file_, print_header_, print_values_,
2097 is_write_committed_, &exec_state_);
2098}
2099
2100// ----------------------------------------------------------------------------
2101
2102GetCommand::GetCommand(const std::vector<std::string>& params,
2103 const std::map<std::string, std::string>& options,
2104 const std::vector<std::string>& flags)
2105 : LDBCommand(
2106 options, flags, true,
2107 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2108 if (params.size() != 1) {
2109 exec_state_ = LDBCommandExecuteResult::Failed(
2110 "<key> must be specified for the get command");
2111 } else {
2112 key_ = params.at(0);
2113 }
2114
2115 if (is_key_hex_) {
2116 key_ = HexToString(key_);
2117 }
2118}
2119
2120void GetCommand::Help(std::string& ret) {
2121 ret.append(" ");
2122 ret.append(GetCommand::Name());
2123 ret.append(" <key>");
2124 ret.append(" [--" + ARG_TTL + "]");
2125 ret.append("\n");
2126}
2127
2128void GetCommand::DoCommand() {
2129 if (!db_) {
2130 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2131 return;
2132 }
2133 std::string value;
2134 Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
2135 if (st.ok()) {
2136 fprintf(stdoutstdout, "%s\n",
2137 (is_value_hex_ ? StringToHex(value) : value).c_str());
2138 } else {
2139 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2140 }
2141}
2142
2143// ----------------------------------------------------------------------------
2144
2145ApproxSizeCommand::ApproxSizeCommand(
2146 const std::vector<std::string>& /*params*/,
2147 const std::map<std::string, std::string>& options,
2148 const std::vector<std::string>& flags)
2149 : LDBCommand(options, flags, true,
2150 BuildCmdLineOptions(
2151 {ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM, ARG_TO})) {
2152 if (options.find(ARG_FROM) != options.end()) {
2153 start_key_ = options.find(ARG_FROM)->second;
2154 } else {
2155 exec_state_ = LDBCommandExecuteResult::Failed(
2156 ARG_FROM + " must be specified for approxsize command");
2157 return;
2158 }
2159
2160 if (options.find(ARG_TO) != options.end()) {
2161 end_key_ = options.find(ARG_TO)->second;
2162 } else {
2163 exec_state_ = LDBCommandExecuteResult::Failed(
2164 ARG_TO + " must be specified for approxsize command");
2165 return;
2166 }
2167
2168 if (is_key_hex_) {
2169 start_key_ = HexToString(start_key_);
2170 end_key_ = HexToString(end_key_);
2171 }
2172}
2173
2174void ApproxSizeCommand::Help(std::string& ret) {
2175 ret.append(" ");
2176 ret.append(ApproxSizeCommand::Name());
2177 ret.append(HelpRangeCmdArgs());
2178 ret.append("\n");
2179}
2180
2181void ApproxSizeCommand::DoCommand() {
2182 if (!db_) {
2183 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2184 return;
2185 }
2186 Range ranges[1];
2187 ranges[0] = Range(start_key_, end_key_);
2188 uint64_t sizes[1];
2189 db_->GetApproximateSizes(GetCfHandle(), ranges, 1, sizes);
2190 fprintf(stdoutstdout, "%lu\n", (unsigned long)sizes[0]);
2191 /* Weird that GetApproximateSizes() returns void, although documentation
2192 * says that it returns a Status object.
2193 if (!st.ok()) {
2194 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2195 }
2196 */
2197}
2198
2199// ----------------------------------------------------------------------------
2200
2201BatchPutCommand::BatchPutCommand(
2202 const std::vector<std::string>& params,
2203 const std::map<std::string, std::string>& options,
2204 const std::vector<std::string>& flags)
2205 : LDBCommand(options, flags, false,
2206 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2207 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2208 if (params.size() < 2) {
2209 exec_state_ = LDBCommandExecuteResult::Failed(
2210 "At least one <key> <value> pair must be specified batchput.");
2211 } else if (params.size() % 2 != 0) {
2212 exec_state_ = LDBCommandExecuteResult::Failed(
2213 "Equal number of <key>s and <value>s must be specified for batchput.");
2214 } else {
2215 for (size_t i = 0; i < params.size(); i += 2) {
2216 std::string key = params.at(i);
2217 std::string value = params.at(i + 1);
2218 key_values_.push_back(std::pair<std::string, std::string>(
2219 is_key_hex_ ? HexToString(key) : key,
2220 is_value_hex_ ? HexToString(value) : value));
2221 }
2222 }
2223 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2224}
2225
2226void BatchPutCommand::Help(std::string& ret) {
2227 ret.append(" ");
2228 ret.append(BatchPutCommand::Name());
2229 ret.append(" <key> <value> [<key> <value>] [..]");
2230 ret.append(" [--" + ARG_TTL + "]");
2231 ret.append("\n");
2232}
2233
2234void BatchPutCommand::DoCommand() {
2235 if (!db_) {
2236 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2237 return;
2238 }
2239 WriteBatch batch;
2240
2241 for (std::vector<std::pair<std::string, std::string>>::const_iterator itr =
2242 key_values_.begin();
2243 itr != key_values_.end(); ++itr) {
2244 batch.Put(GetCfHandle(), itr->first, itr->second);
2245 }
2246 Status st = db_->Write(WriteOptions(), &batch);
2247 if (st.ok()) {
2248 fprintf(stdoutstdout, "OK\n");
2249 } else {
2250 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2251 }
2252}
2253
2254Options BatchPutCommand::PrepareOptionsForOpenDB() {
2255 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2256 opt.create_if_missing = create_if_missing_;
2257 return opt;
2258}
2259
2260// ----------------------------------------------------------------------------
2261
2262ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/,
2263 const std::map<std::string, std::string>& options,
2264 const std::vector<std::string>& flags)
2265 : LDBCommand(
2266 options, flags, true,
2267 BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
2268 ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
2269 ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
2270 start_key_specified_(false),
2271 end_key_specified_(false),
2272 max_keys_scanned_(-1),
2273 no_value_(false) {
2274 std::map<std::string, std::string>::const_iterator itr =
2275 options.find(ARG_FROM);
2276 if (itr != options.end()) {
2277 start_key_ = itr->second;
2278 if (is_key_hex_) {
2279 start_key_ = HexToString(start_key_);
2280 }
2281 start_key_specified_ = true;
2282 }
2283 itr = options.find(ARG_TO);
2284 if (itr != options.end()) {
2285 end_key_ = itr->second;
2286 if (is_key_hex_) {
2287 end_key_ = HexToString(end_key_);
2288 }
2289 end_key_specified_ = true;
2290 }
2291
2292 std::vector<std::string>::const_iterator vitr =
2293 std::find(flags.begin(), flags.end(), ARG_NO_VALUE);
2294 if (vitr != flags.end()) {
2295 no_value_ = true;
2296 }
2297
2298 itr = options.find(ARG_MAX_KEYS);
2299 if (itr != options.end()) {
2300 try {
2301#if defined(CYGWIN)
2302 max_keys_scanned_ = strtol(itr->second.c_str(), 0, 10);
2303#else
2304 max_keys_scanned_ = std::stoi(itr->second);
2305#endif
2306 } catch (const std::invalid_argument&) {
2307 exec_state_ = LDBCommandExecuteResult::Failed(ARG_MAX_KEYS +
2308 " has an invalid value");
2309 } catch (const std::out_of_range&) {
2310 exec_state_ = LDBCommandExecuteResult::Failed(
2311 ARG_MAX_KEYS + " has a value out-of-range");
2312 }
2313 }
2314}
2315
2316void ScanCommand::Help(std::string& ret) {
2317 ret.append(" ");
2318 ret.append(ScanCommand::Name());
2319 ret.append(HelpRangeCmdArgs());
2320 ret.append(" [--" + ARG_TTL + "]");
2321 ret.append(" [--" + ARG_TIMESTAMP + "]");
2322 ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
2323 ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
2324 ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
2325 ret.append(" [--" + ARG_NO_VALUE + "]");
2326 ret.append("\n");
2327}
2328
2329void ScanCommand::DoCommand() {
2330 if (!db_) {
2331 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2332 return;
2333 }
2334
2335 int num_keys_scanned = 0;
2336 ReadOptions scan_read_opts;
2337 scan_read_opts.total_order_seek = true;
2338 Iterator* it = db_->NewIterator(scan_read_opts, GetCfHandle());
2339 if (start_key_specified_) {
2340 it->Seek(start_key_);
2341 } else {
2342 it->SeekToFirst();
2343 }
2344 int ttl_start;
2345 if (!ParseIntOption(option_map_, ARG_TTL_START, ttl_start, exec_state_)) {
2346 ttl_start = DBWithTTLImpl::kMinTimestamp; // TTL introduction time
2347 }
2348 int ttl_end;
2349 if (!ParseIntOption(option_map_, ARG_TTL_END, ttl_end, exec_state_)) {
2350 ttl_end = DBWithTTLImpl::kMaxTimestamp; // Max time allowed by TTL feature
2351 }
2352 if (ttl_end < ttl_start) {
2353 fprintf(stderrstderr, "Error: End time can't be less than start time\n");
2354 delete it;
2355 return;
2356 }
2357 if (is_db_ttl_ && timestamp_) {
2358 fprintf(stdoutstdout, "Scanning key-values from %s to %s\n",
2359 ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
2360 }
2361 for ( ;
2362 it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
2363 it->Next()) {
2364 if (is_db_ttl_) {
2365 TtlIterator* it_ttl = static_cast_with_check<TtlIterator, Iterator>(it);
2366 int rawtime = it_ttl->timestamp();
2367 if (rawtime < ttl_start || rawtime >= ttl_end) {
2368 continue;
2369 }
2370 if (timestamp_) {
2371 fprintf(stdoutstdout, "%s ", ReadableTime(rawtime).c_str());
2372 }
2373 }
2374
2375 Slice key_slice = it->key();
2376
2377 std::string formatted_key;
2378 if (is_key_hex_) {
2379 formatted_key = "0x" + key_slice.ToString(true /* hex */);
2380 key_slice = formatted_key;
2381 } else if (ldb_options_.key_formatter) {
2382 formatted_key = ldb_options_.key_formatter->Format(key_slice);
2383 key_slice = formatted_key;
2384 }
2385
2386 if (no_value_) {
2387 fprintf(stdoutstdout, "%.*s\n", static_cast<int>(key_slice.size()),
2388 key_slice.data());
2389 } else {
2390 Slice val_slice = it->value();
2391 std::string formatted_value;
2392 if (is_value_hex_) {
2393 formatted_value = "0x" + val_slice.ToString(true /* hex */);
2394 val_slice = formatted_value;
2395 }
2396 fprintf(stdoutstdout, "%.*s : %.*s\n", static_cast<int>(key_slice.size()),
2397 key_slice.data(), static_cast<int>(val_slice.size()),
2398 val_slice.data());
2399 }
2400
2401 num_keys_scanned++;
2402 if (max_keys_scanned_ >= 0 && num_keys_scanned >= max_keys_scanned_) {
2403 break;
2404 }
2405 }
2406 if (!it->status().ok()) { // Check for any errors found during the scan
2407 exec_state_ = LDBCommandExecuteResult::Failed(it->status().ToString());
2408 }
2409 delete it;
2410}
2411
2412// ----------------------------------------------------------------------------
2413
2414DeleteCommand::DeleteCommand(const std::vector<std::string>& params,
2415 const std::map<std::string, std::string>& options,
2416 const std::vector<std::string>& flags)
2417 : LDBCommand(options, flags, false,
2418 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2419 if (params.size() != 1) {
2420 exec_state_ = LDBCommandExecuteResult::Failed(
2421 "KEY must be specified for the delete command");
2422 } else {
2423 key_ = params.at(0);
2424 if (is_key_hex_) {
2425 key_ = HexToString(key_);
2426 }
2427 }
2428}
2429
2430void DeleteCommand::Help(std::string& ret) {
2431 ret.append(" ");
2432 ret.append(DeleteCommand::Name() + " <key>");
2433 ret.append("\n");
2434}
2435
2436void DeleteCommand::DoCommand() {
2437 if (!db_) {
2438 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2439 return;
2440 }
2441 Status st = db_->Delete(WriteOptions(), GetCfHandle(), key_);
2442 if (st.ok()) {
2443 fprintf(stdoutstdout, "OK\n");
2444 } else {
2445 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2446 }
2447}
2448
2449DeleteRangeCommand::DeleteRangeCommand(
2450 const std::vector<std::string>& params,
2451 const std::map<std::string, std::string>& options,
2452 const std::vector<std::string>& flags)
2453 : LDBCommand(options, flags, false,
2454 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2455 if (params.size() != 2) {
2456 exec_state_ = LDBCommandExecuteResult::Failed(
2457 "begin and end keys must be specified for the delete command");
2458 } else {
2459 begin_key_ = params.at(0);
2460 end_key_ = params.at(1);
2461 if (is_key_hex_) {
2462 begin_key_ = HexToString(begin_key_);
2463 end_key_ = HexToString(end_key_);
2464 }
2465 }
2466}
2467
2468void DeleteRangeCommand::Help(std::string& ret) {
2469 ret.append(" ");
2470 ret.append(DeleteRangeCommand::Name() + " <begin key> <end key>");
2471 ret.append("\n");
2472}
2473
2474void DeleteRangeCommand::DoCommand() {
2475 if (!db_) {
2476 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2477 return;
2478 }
2479 Status st =
2480 db_->DeleteRange(WriteOptions(), GetCfHandle(), begin_key_, end_key_);
2481 if (st.ok()) {
2482 fprintf(stdoutstdout, "OK\n");
2483 } else {
2484 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2485 }
2486}
2487
2488PutCommand::PutCommand(const std::vector<std::string>& params,
2489 const std::map<std::string, std::string>& options,
2490 const std::vector<std::string>& flags)
2491 : LDBCommand(options, flags, false,
2492 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
2493 ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) {
2494 if (params.size() != 2) {
2495 exec_state_ = LDBCommandExecuteResult::Failed(
2496 "<key> and <value> must be specified for the put command");
2497 } else {
2498 key_ = params.at(0);
2499 value_ = params.at(1);
2500 }
2501
2502 if (is_key_hex_) {
2503 key_ = HexToString(key_);
2504 }
2505
2506 if (is_value_hex_) {
2507 value_ = HexToString(value_);
2508 }
2509 create_if_missing_ = IsFlagPresent(flags_, ARG_CREATE_IF_MISSING);
2510}
2511
2512void PutCommand::Help(std::string& ret) {
2513 ret.append(" ");
2514 ret.append(PutCommand::Name());
2515 ret.append(" <key> <value> ");
2516 ret.append(" [--" + ARG_TTL + "]");
2517 ret.append("\n");
2518}
2519
2520void PutCommand::DoCommand() {
2521 if (!db_) {
2522 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2523 return;
2524 }
2525 Status st = db_->Put(WriteOptions(), GetCfHandle(), key_, value_);
2526 if (st.ok()) {
2527 fprintf(stdoutstdout, "OK\n");
2528 } else {
2529 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2530 }
2531}
2532
2533Options PutCommand::PrepareOptionsForOpenDB() {
2534 Options opt = LDBCommand::PrepareOptionsForOpenDB();
2535 opt.create_if_missing = create_if_missing_;
2536 return opt;
2537}
2538
2539// ----------------------------------------------------------------------------
2540
2541const char* DBQuerierCommand::HELP_CMD = "help";
2542const char* DBQuerierCommand::GET_CMD = "get";
2543const char* DBQuerierCommand::PUT_CMD = "put";
2544const char* DBQuerierCommand::DELETE_CMD = "delete";
2545
2546DBQuerierCommand::DBQuerierCommand(
2547 const std::vector<std::string>& /*params*/,
2548 const std::map<std::string, std::string>& options,
2549 const std::vector<std::string>& flags)
2550 : LDBCommand(
2551 options, flags, false,
2552 BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
2553
2554}
2555
2556void DBQuerierCommand::Help(std::string& ret) {
2557 ret.append(" ");
2558 ret.append(DBQuerierCommand::Name());
2559 ret.append(" [--" + ARG_TTL + "]");
2560 ret.append("\n");
2561 ret.append(" Starts a REPL shell. Type help for list of available "
2562 "commands.");
2563 ret.append("\n");
2564}
2565
2566void DBQuerierCommand::DoCommand() {
2567 if (!db_) {
2568 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2569 return;
2570 }
2571
2572 ReadOptions read_options;
2573 WriteOptions write_options;
2574
2575 std::string line;
2576 std::string key;
2577 std::string value;
2578 while (getline(std::cin, line, '\n')) {
2579 // Parse line into std::vector<std::string>
2580 std::vector<std::string> tokens;
2581 size_t pos = 0;
2582 while (true) {
2583 size_t pos2 = line.find(' ', pos);
2584 if (pos2 == std::string::npos) {
2585 break;
2586 }
2587 tokens.push_back(line.substr(pos, pos2-pos));
2588 pos = pos2 + 1;
2589 }
2590 tokens.push_back(line.substr(pos));
2591
2592 const std::string& cmd = tokens[0];
2593
2594 if (cmd == HELP_CMD) {
2595 fprintf(stdoutstdout,
2596 "get <key>\n"
2597 "put <key> <value>\n"
2598 "delete <key>\n");
2599 } else if (cmd == DELETE_CMD && tokens.size() == 2) {
2600 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2601 db_->Delete(write_options, GetCfHandle(), Slice(key));
2602 fprintf(stdoutstdout, "Successfully deleted %s\n", tokens[1].c_str());
2603 } else if (cmd == PUT_CMD && tokens.size() == 3) {
2604 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2605 value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
2606 db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
2607 fprintf(stdoutstdout, "Successfully put %s %s\n",
2608 tokens[1].c_str(), tokens[2].c_str());
2609 } else if (cmd == GET_CMD && tokens.size() == 2) {
2610 key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
2611 if (db_->Get(read_options, GetCfHandle(), Slice(key), &value).ok()) {
2612 fprintf(stdoutstdout, "%s\n", PrintKeyValue(key, value,
2613 is_key_hex_, is_value_hex_).c_str());
2614 } else {
2615 fprintf(stdoutstdout, "Not found %s\n", tokens[1].c_str());
2616 }
2617 } else {
2618 fprintf(stdoutstdout, "Unknown command %s\n", line.c_str());
2619 }
2620 }
2621}
2622
2623// ----------------------------------------------------------------------------
2624
2625CheckConsistencyCommand::CheckConsistencyCommand(
2626 const std::vector<std::string>& /*params*/,
2627 const std::map<std::string, std::string>& options,
2628 const std::vector<std::string>& flags)
2629 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2630
2631void CheckConsistencyCommand::Help(std::string& ret) {
2632 ret.append(" ");
2633 ret.append(CheckConsistencyCommand::Name());
2634 ret.append("\n");
2635}
2636
2637void CheckConsistencyCommand::DoCommand() {
2638 Options opt = PrepareOptionsForOpenDB();
2639 opt.paranoid_checks = true;
2640 if (!exec_state_.IsNotStarted()) {
2641 return;
2642 }
2643 DB* db;
2644 Status st = DB::OpenForReadOnly(opt, db_path_, &db, false);
2645 delete db;
2646 if (st.ok()) {
2647 fprintf(stdoutstdout, "OK\n");
2648 } else {
2649 exec_state_ = LDBCommandExecuteResult::Failed(st.ToString());
2650 }
2651}
2652
2653// ----------------------------------------------------------------------------
2654
2655const std::string CheckPointCommand::ARG_CHECKPOINT_DIR = "checkpoint_dir";
2656
2657CheckPointCommand::CheckPointCommand(
2658 const std::vector<std::string>& /*params*/,
2659 const std::map<std::string, std::string>& options,
2660 const std::vector<std::string>& flags)
2661 : LDBCommand(options, flags, false /* is_read_only */,
2662 BuildCmdLineOptions({ARG_CHECKPOINT_DIR})) {
2663 auto itr = options.find(ARG_CHECKPOINT_DIR);
2664 if (itr != options.end()) {
2665 checkpoint_dir_ = itr->second;
2666 }
2667}
2668
2669void CheckPointCommand::Help(std::string& ret) {
2670 ret.append(" ");
2671 ret.append(CheckPointCommand::Name());
2672 ret.append(" [--" + ARG_CHECKPOINT_DIR + "] ");
2673 ret.append("\n");
2674}
2675
2676void CheckPointCommand::DoCommand() {
2677 if (!db_) {
2678 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2679 return;
2680 }
2681 Checkpoint* checkpoint;
2682 Status status = Checkpoint::Create(db_, &checkpoint);
2683 status = checkpoint->CreateCheckpoint(checkpoint_dir_);
2684 if (status.ok()) {
2685 printf("OK\n");
2686 } else {
2687 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2688 }
2689}
2690
2691// ----------------------------------------------------------------------------
2692
2693RepairCommand::RepairCommand(const std::vector<std::string>& /*params*/,
2694 const std::map<std::string, std::string>& options,
2695 const std::vector<std::string>& flags)
2696 : LDBCommand(options, flags, false, BuildCmdLineOptions({})) {}
2697
2698void RepairCommand::Help(std::string& ret) {
2699 ret.append(" ");
2700 ret.append(RepairCommand::Name());
2701 ret.append("\n");
2702}
2703
2704void RepairCommand::DoCommand() {
2705 Options options = PrepareOptionsForOpenDB();
2706 options.info_log.reset(new StderrLogger(InfoLogLevel::WARN_LEVEL));
2707 Status status = RepairDB(db_path_, options);
2708 if (status.ok()) {
2709 printf("OK\n");
2710 } else {
2711 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2712 }
2713}
2714
2715// ----------------------------------------------------------------------------
2716
2717const std::string BackupableCommand::ARG_NUM_THREADS = "num_threads";
2718const std::string BackupableCommand::ARG_BACKUP_ENV_URI = "backup_env_uri";
2719const std::string BackupableCommand::ARG_BACKUP_DIR = "backup_dir";
2720const std::string BackupableCommand::ARG_STDERR_LOG_LEVEL = "stderr_log_level";
2721
2722BackupableCommand::BackupableCommand(
2723 const std::vector<std::string>& /*params*/,
2724 const std::map<std::string, std::string>& options,
2725 const std::vector<std::string>& flags)
2726 : LDBCommand(options, flags, false /* is_read_only */,
2727 BuildCmdLineOptions({ARG_BACKUP_ENV_URI, ARG_BACKUP_DIR,
2728 ARG_NUM_THREADS, ARG_STDERR_LOG_LEVEL})),
2729 num_threads_(1) {
2730 auto itr = options.find(ARG_NUM_THREADS);
2731 if (itr != options.end()) {
2732 num_threads_ = std::stoi(itr->second);
2733 }
2734 itr = options.find(ARG_BACKUP_ENV_URI);
2735 if (itr != options.end()) {
2736 backup_env_uri_ = itr->second;
2737 }
2738 itr = options.find(ARG_BACKUP_DIR);
2739 if (itr == options.end()) {
2740 exec_state_ = LDBCommandExecuteResult::Failed("--" + ARG_BACKUP_DIR +
2741 ": missing backup directory");
2742 } else {
2743 backup_dir_ = itr->second;
2744 }
2745
2746 itr = options.find(ARG_STDERR_LOG_LEVEL);
2747 if (itr != options.end()) {
2748 int stderr_log_level = std::stoi(itr->second);
2749 if (stderr_log_level < 0 ||
2750 stderr_log_level >= InfoLogLevel::NUM_INFO_LOG_LEVELS) {
2751 exec_state_ = LDBCommandExecuteResult::Failed(
2752 ARG_STDERR_LOG_LEVEL + " must be >= 0 and < " +
2753 std::to_string(InfoLogLevel::NUM_INFO_LOG_LEVELS) + ".");
2754 } else {
2755 logger_.reset(
2756 new StderrLogger(static_cast<InfoLogLevel>(stderr_log_level)));
2757 }
2758 }
2759}
2760
2761void BackupableCommand::Help(const std::string& name, std::string& ret) {
2762 ret.append(" ");
2763 ret.append(name);
2764 ret.append(" [--" + ARG_BACKUP_ENV_URI + "] ");
2765 ret.append(" [--" + ARG_BACKUP_DIR + "] ");
2766 ret.append(" [--" + ARG_NUM_THREADS + "] ");
2767 ret.append(" [--" + ARG_STDERR_LOG_LEVEL + "=<int (InfoLogLevel)>] ");
2768 ret.append("\n");
2769}
2770
2771// ----------------------------------------------------------------------------
2772
2773BackupCommand::BackupCommand(const std::vector<std::string>& params,
2774 const std::map<std::string, std::string>& options,
2775 const std::vector<std::string>& flags)
2776 : BackupableCommand(params, options, flags) {}
2777
2778void BackupCommand::Help(std::string& ret) {
2779 BackupableCommand::Help(Name(), ret);
2780}
2781
2782void BackupCommand::DoCommand() {
2783 BackupEngine* backup_engine;
2784 Status status;
2785 if (!db_) {
2786 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2787 return;
2788 }
2789 printf("open db OK\n");
2790 std::unique_ptr<Env> custom_env_guard;
2791 Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2792 BackupableDBOptions backup_options =
2793 BackupableDBOptions(backup_dir_, custom_env);
2794 backup_options.info_log = logger_.get();
2795 backup_options.max_background_operations = num_threads_;
2796 status = BackupEngine::Open(Env::Default(), backup_options, &backup_engine);
2797 if (status.ok()) {
2798 printf("open backup engine OK\n");
2799 } else {
2800 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2801 return;
2802 }
2803 status = backup_engine->CreateNewBackup(db_);
2804 if (status.ok()) {
2805 printf("create new backup OK\n");
2806 } else {
2807 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2808 return;
2809 }
2810}
2811
2812// ----------------------------------------------------------------------------
2813
2814RestoreCommand::RestoreCommand(
2815 const std::vector<std::string>& params,
2816 const std::map<std::string, std::string>& options,
2817 const std::vector<std::string>& flags)
2818 : BackupableCommand(params, options, flags) {}
2819
2820void RestoreCommand::Help(std::string& ret) {
2821 BackupableCommand::Help(Name(), ret);
2822}
2823
2824void RestoreCommand::DoCommand() {
2825 std::unique_ptr<Env> custom_env_guard;
2826 Env* custom_env = NewCustomObject<Env>(backup_env_uri_, &custom_env_guard);
2827 std::unique_ptr<BackupEngineReadOnly> restore_engine;
2828 Status status;
2829 {
2830 BackupableDBOptions opts(backup_dir_, custom_env);
2831 opts.info_log = logger_.get();
2832 opts.max_background_operations = num_threads_;
2833 BackupEngineReadOnly* raw_restore_engine_ptr;
2834 status = BackupEngineReadOnly::Open(Env::Default(), opts,
2835 &raw_restore_engine_ptr);
2836 if (status.ok()) {
2837 restore_engine.reset(raw_restore_engine_ptr);
2838 }
2839 }
2840 if (status.ok()) {
2841 printf("open restore engine OK\n");
2842 status = restore_engine->RestoreDBFromLatestBackup(db_path_, db_path_);
2843 }
2844 if (status.ok()) {
2845 printf("restore from backup OK\n");
2846 } else {
2847 exec_state_ = LDBCommandExecuteResult::Failed(status.ToString());
2848 }
2849}
2850
2851// ----------------------------------------------------------------------------
2852
2853namespace {
2854
2855void DumpSstFile(Options options, std::string filename, bool output_hex,
2856 bool show_properties) {
2857 std::string from_key;
2858 std::string to_key;
2859 if (filename.length() <= 4 ||
2860 filename.rfind(".sst") != filename.length() - 4) {
2861 std::cout << "Invalid sst file name." << std::endl;
2862 return;
2863 }
2864 // no verification
2865 rocksdb::SstFileDumper dumper(options, filename, false, output_hex);
2866 Status st = dumper.ReadSequential(true, std::numeric_limits<uint64_t>::max(),
2867 false, // has_from
2868 from_key, false, // has_to
2869 to_key);
2870 if (!st.ok()) {
2871 std::cerr << "Error in reading SST file " << filename << st.ToString()
2872 << std::endl;
2873 return;
2874 }
2875
2876 if (show_properties) {
2877 const rocksdb::TableProperties* table_properties;
2878
2879 std::shared_ptr<const rocksdb::TableProperties>
2880 table_properties_from_reader;
2881 st = dumper.ReadTableProperties(&table_properties_from_reader);
2882 if (!st.ok()) {
2883 std::cerr << filename << ": " << st.ToString()
2884 << ". Try to use initial table properties" << std::endl;
2885 table_properties = dumper.GetInitTableProperties();
2886 } else {
2887 table_properties = table_properties_from_reader.get();
2888 }
2889 if (table_properties != nullptr) {
2890 std::cout << std::endl << "Table Properties:" << std::endl;
2891 std::cout << table_properties->ToString("\n") << std::endl;
2892 }
2893 }
2894}
2895
2896} // namespace
2897
2898DBFileDumperCommand::DBFileDumperCommand(
2899 const std::vector<std::string>& /*params*/,
2900 const std::map<std::string, std::string>& options,
2901 const std::vector<std::string>& flags)
2902 : LDBCommand(options, flags, true, BuildCmdLineOptions({})) {}
2903
2904void DBFileDumperCommand::Help(std::string& ret) {
2905 ret.append(" ");
2906 ret.append(DBFileDumperCommand::Name());
2907 ret.append("\n");
2908}
2909
2910void DBFileDumperCommand::DoCommand() {
2911 if (!db_) {
2912 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2913 return;
2914 }
2915 Status s;
2916
2917 std::cout << "Manifest File" << std::endl;
2918 std::cout << "==============================" << std::endl;
2919 std::string manifest_filename;
2920 s = ReadFileToString(db_->GetEnv(), CurrentFileName(db_->GetName()),
2921 &manifest_filename);
2922 if (!s.ok() || manifest_filename.empty() ||
2923 manifest_filename.back() != '\n') {
2924 std::cerr << "Error when reading CURRENT file "
2925 << CurrentFileName(db_->GetName()) << std::endl;
2926 }
2927 // remove the trailing '\n'
2928 manifest_filename.resize(manifest_filename.size() - 1);
2929 std::string manifest_filepath = db_->GetName() + "/" + manifest_filename;
2930 std::cout << manifest_filepath << std::endl;
2931 DumpManifestFile(options_, manifest_filepath, false, false, false);
2932 std::cout << std::endl;
2933
2934 std::cout << "SST Files" << std::endl;
2935 std::cout << "==============================" << std::endl;
2936 std::vector<LiveFileMetaData> metadata;
2937 db_->GetLiveFilesMetaData(&metadata);
2938 for (auto& fileMetadata : metadata) {
2939 std::string filename = fileMetadata.db_path + fileMetadata.name;
2940 std::cout << filename << " level:" << fileMetadata.level << std::endl;
2941 std::cout << "------------------------------" << std::endl;
2942 DumpSstFile(options_, filename, false, true);
2943 std::cout << std::endl;
2944 }
2945 std::cout << std::endl;
2946
2947 std::cout << "Write Ahead Log Files" << std::endl;
2948 std::cout << "==============================" << std::endl;
2949 rocksdb::VectorLogPtr wal_files;
2950 s = db_->GetSortedWalFiles(wal_files);
2951 if (!s.ok()) {
2952 std::cerr << "Error when getting WAL files" << std::endl;
2953 } else {
2954 for (auto& wal : wal_files) {
2955 // TODO(qyang): option.wal_dir should be passed into ldb command
2956 std::string filename = db_->GetOptions().wal_dir + wal->PathName();
2957 std::cout << filename << std::endl;
2958 // TODO(myabandeh): allow configuring is_write_commited
2959 DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
2960 &exec_state_);
2961 }
2962 }
2963}
2964
2965void WriteExternalSstFilesCommand::Help(std::string& ret) {
2966 ret.append(" ");
2967 ret.append(WriteExternalSstFilesCommand::Name());
2968 ret.append(" <output_sst_path>");
2969 ret.append("\n");
2970}
2971
2972WriteExternalSstFilesCommand::WriteExternalSstFilesCommand(
2973 const std::vector<std::string>& params,
2974 const std::map<std::string, std::string>& options,
2975 const std::vector<std::string>& flags)
2976 : LDBCommand(
2977 options, flags, false /* is_read_only */,
2978 BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_FROM,
2979 ARG_TO, ARG_CREATE_IF_MISSING})) {
2980 create_if_missing_ =
2981 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
2982 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
2983 if (params.size() != 1) {
2984 exec_state_ = LDBCommandExecuteResult::Failed(
2985 "output SST file path must be specified");
2986 } else {
2987 output_sst_path_ = params.at(0);
2988 }
2989}
2990
2991void WriteExternalSstFilesCommand::DoCommand() {
2992 if (!db_) {
2993 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
2994 return;
2995 }
2996 ColumnFamilyHandle* cfh = GetCfHandle();
2997 SstFileWriter sst_file_writer(EnvOptions(), db_->GetOptions(), cfh);
2998 Status status = sst_file_writer.Open(output_sst_path_);
2999 if (!status.ok()) {
3000 exec_state_ = LDBCommandExecuteResult::Failed("failed to open SST file: " +
3001 status.ToString());
3002 return;
3003 }
3004
3005 int bad_lines = 0;
3006 std::string line;
3007 std::ifstream ifs_stdin("/dev/stdin");
3008 std::istream* istream_p = ifs_stdin.is_open() ? &ifs_stdin : &std::cin;
3009 while (getline(*istream_p, line, '\n')) {
3010 std::string key;
3011 std::string value;
3012 if (ParseKeyValue(line, &key, &value, is_key_hex_, is_value_hex_)) {
3013 status = sst_file_writer.Put(key, value);
3014 if (!status.ok()) {
3015 exec_state_ = LDBCommandExecuteResult::Failed(
3016 "failed to write record to file: " + status.ToString());
3017 return;
3018 }
3019 } else if (0 == line.find("Keys in range:")) {
3020 // ignore this line
3021 } else if (0 == line.find("Created bg thread 0x")) {
3022 // ignore this line
3023 } else {
3024 bad_lines++;
3025 }
3026 }
3027
3028 status = sst_file_writer.Finish();
3029 if (!status.ok()) {
3030 exec_state_ = LDBCommandExecuteResult::Failed(
3031 "Failed to finish writing to file: " + status.ToString());
3032 return;
3033 }
3034
3035 if (bad_lines > 0) {
3036 fprintf(stderrstderr, "Warning: %d bad lines ignored.\n", bad_lines);
3037 }
3038 exec_state_ = LDBCommandExecuteResult::Succeed(
3039 "external SST file written to " + output_sst_path_);
3040}
3041
3042Options WriteExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3043 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3044 opt.create_if_missing = create_if_missing_;
3045 return opt;
3046}
3047
3048const std::string IngestExternalSstFilesCommand::ARG_MOVE_FILES = "move_files";
3049const std::string IngestExternalSstFilesCommand::ARG_SNAPSHOT_CONSISTENCY =
3050 "snapshot_consistency";
3051const std::string IngestExternalSstFilesCommand::ARG_ALLOW_GLOBAL_SEQNO =
3052 "allow_global_seqno";
3053const std::string IngestExternalSstFilesCommand::ARG_ALLOW_BLOCKING_FLUSH =
3054 "allow_blocking_flush";
3055const std::string IngestExternalSstFilesCommand::ARG_INGEST_BEHIND =
3056 "ingest_behind";
3057const std::string IngestExternalSstFilesCommand::ARG_WRITE_GLOBAL_SEQNO =
3058 "write_global_seqno";
3059
3060void IngestExternalSstFilesCommand::Help(std::string& ret) {
3061 ret.append(" ");
3062 ret.append(IngestExternalSstFilesCommand::Name());
3063 ret.append(" <input_sst_path>");
3064 ret.append(" [--" + ARG_MOVE_FILES + "] ");
3065 ret.append(" [--" + ARG_SNAPSHOT_CONSISTENCY + "] ");
3066 ret.append(" [--" + ARG_ALLOW_GLOBAL_SEQNO + "] ");
3067 ret.append(" [--" + ARG_ALLOW_BLOCKING_FLUSH + "] ");
3068 ret.append(" [--" + ARG_INGEST_BEHIND + "] ");
3069 ret.append(" [--" + ARG_WRITE_GLOBAL_SEQNO + "] ");
3070 ret.append("\n");
3071}
3072
3073IngestExternalSstFilesCommand::IngestExternalSstFilesCommand(
3074 const std::vector<std::string>& params,
3075 const std::map<std::string, std::string>& options,
3076 const std::vector<std::string>& flags)
3077 : LDBCommand(
3078 options, flags, false /* is_read_only */,
3079 BuildCmdLineOptions({ARG_MOVE_FILES, ARG_SNAPSHOT_CONSISTENCY,
3080 ARG_ALLOW_GLOBAL_SEQNO, ARG_CREATE_IF_MISSING,
3081 ARG_ALLOW_BLOCKING_FLUSH, ARG_INGEST_BEHIND,
3082 ARG_WRITE_GLOBAL_SEQNO})),
3083 move_files_(false),
3084 snapshot_consistency_(true),
3085 allow_global_seqno_(true),
3086 allow_blocking_flush_(true),
3087 ingest_behind_(false),
3088 write_global_seqno_(true) {
3089 create_if_missing_ =
3090 IsFlagPresent(flags, ARG_CREATE_IF_MISSING) ||
3091 ParseBooleanOption(options, ARG_CREATE_IF_MISSING, false);
3092 move_files_ = IsFlagPresent(flags, ARG_MOVE_FILES) ||
3093 ParseBooleanOption(options, ARG_MOVE_FILES, false);
3094 snapshot_consistency_ =
3095 IsFlagPresent(flags, ARG_SNAPSHOT_CONSISTENCY) ||
3096 ParseBooleanOption(options, ARG_SNAPSHOT_CONSISTENCY, true);
3097 allow_global_seqno_ =
3098 IsFlagPresent(flags, ARG_ALLOW_GLOBAL_SEQNO) ||
3099 ParseBooleanOption(options, ARG_ALLOW_GLOBAL_SEQNO, true);
3100 allow_blocking_flush_ =
3101 IsFlagPresent(flags, ARG_ALLOW_BLOCKING_FLUSH) ||
3102 ParseBooleanOption(options, ARG_ALLOW_BLOCKING_FLUSH, true);
3103 ingest_behind_ = IsFlagPresent(flags, ARG_INGEST_BEHIND) ||
3104 ParseBooleanOption(options, ARG_INGEST_BEHIND, false);
3105 write_global_seqno_ =
3106 IsFlagPresent(flags, ARG_WRITE_GLOBAL_SEQNO) ||
3107 ParseBooleanOption(options, ARG_WRITE_GLOBAL_SEQNO, true);
3108
3109 if (allow_global_seqno_) {
3110 if (!write_global_seqno_) {
3111 fprintf(stderrstderr,
3112 "Warning: not writing global_seqno to the ingested SST can\n"
3113 "prevent older versions of RocksDB from being able to open it\n");
3114 }
3115 } else {
3116 if (write_global_seqno_) {
3117 exec_state_ = LDBCommandExecuteResult::Failed(
3118 "ldb cannot write global_seqno to the ingested SST when global_seqno "
3119 "is not allowed");
3120 }
3121 }
3122
3123 if (params.size() != 1) {
3124 exec_state_ =
3125 LDBCommandExecuteResult::Failed("input SST path must be specified");
3126 } else {
3127 input_sst_path_ = params.at(0);
3128 }
3129}
3130
3131void IngestExternalSstFilesCommand::DoCommand() {
3132 if (!db_) {
3133 assert(GetExecuteState().IsFailed())(static_cast<void> (0));
3134 return;
3135 }
3136 if (GetExecuteState().IsFailed()) {
3137 return;
3138 }
3139 ColumnFamilyHandle* cfh = GetCfHandle();
3140 IngestExternalFileOptions ifo;
3141 ifo.move_files = move_files_;
3142 ifo.snapshot_consistency = snapshot_consistency_;
3143 ifo.allow_global_seqno = allow_global_seqno_;
3144 ifo.allow_blocking_flush = allow_blocking_flush_;
3145 ifo.ingest_behind = ingest_behind_;
3146 ifo.write_global_seqno = write_global_seqno_;
3147 Status status = db_->IngestExternalFile(cfh, {input_sst_path_}, ifo);
3148 if (!status.ok()) {
3149 exec_state_ = LDBCommandExecuteResult::Failed(
3150 "failed to ingest external SST: " + status.ToString());
3151 } else {
3152 exec_state_ =
3153 LDBCommandExecuteResult::Succeed("external SST files ingested");
3154 }
3155}
3156
3157Options IngestExternalSstFilesCommand::PrepareOptionsForOpenDB() {
3158 Options opt = LDBCommand::PrepareOptionsForOpenDB();
3159 opt.create_if_missing = create_if_missing_;
3160 return opt;
3161}
3162
3163} // namespace rocksdb
3164#endif // ROCKSDB_LITE