Skip to content

Commit af53218

Browse files
nadavMiznadavMiz
authored
Merge pull request #8925 from nadavMiz/lifecycle-continue-argument
NC | lifecycle | continue last run
2 parents fd284f7 + 73edf4c commit af53218

File tree

6 files changed

+206
-44
lines changed

6 files changed

+206
-44
lines changed

‎src/cmd/manage_nsfs.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -889,8 +889,9 @@ async function lifecycle_management(args) {
889889
const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
890890
const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
891891
const short_status = get_boolean_or_string_value(args.short_status);
892+
const should_continue_last_run = get_boolean_or_string_value(args.continue);
892893
try {
893-
const options = { disable_service_validation, disable_runtime_validation, short_status };
894+
const options = { disable_service_validation, disable_runtime_validation, short_status, should_continue_last_run };
894895
const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options);
895896
if (should_run) {
896897
write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);

‎src/manage_nsfs/health.js

Lines changed: 4 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33

44
const dbg = require('../util/debug_module')(__filename);
55
const _ = require('lodash');
6-
const path = require('path');
76
const P = require('../util/promise');
87
const config = require('../../config');
98
const os_util = require('../util/os_utils');
@@ -18,6 +17,7 @@ const { get_boolean_or_string_value, throw_cli_error, write_stdout_response,
1817
const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
1918
const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError;
2019
const notifications_util = require('../util/notifications_util');
20+
const lifecycle_utils = require('../util/lifecycle_utils');
2121

2222

2323
const HOSTNAME = 'localhost';
@@ -462,7 +462,9 @@ class NSFSHealth {
462462
* @returns {Promise<object>}
463463
*/
464464
async get_lifecycle_health_status() {
465-
const latest_lifecycle_run_status = await this.get_latest_lifecycle_run_status({ silent_if_missing: true });
465+
const latest_lifecycle_run_status = await lifecycle_utils.get_latest_nc_lifecycle_run_status(
466+
this.config_fs,
467+
{ silent_if_missing: true });
466468
if (!latest_lifecycle_run_status) return {};
467469
return {
468470
total_stats: latest_lifecycle_run_status.total_stats,
@@ -471,29 +473,6 @@ class NSFSHealth {
471473
};
472474
}
473475

474-
475-
/**
476-
* get_latest_lifecycle_run_status returns the latest lifecycle run status
477-
* latest run can be found by maxing the lifecycle log entry names, log entry name is the lifecycle_run_{timestamp}.json of the run
478-
* @params {{silent_if_missing: boolean}} options
479-
* @returns {Promise<object | undefined >}
480-
*/
481-
async get_latest_lifecycle_run_status(options) {
482-
const { silent_if_missing = false } = options;
483-
try {
484-
const lifecycle_log_entries = await nb_native().fs.readdir(this.config_fs.fs_context, config.NC_LIFECYCLE_LOGS_DIR);
485-
const latest_lifecycle_run = _.maxBy(lifecycle_log_entries, entry => entry.name);
486-
const latest_lifecycle_run_status_path = path.join(config.NC_LIFECYCLE_LOGS_DIR, latest_lifecycle_run.name);
487-
const latest_lifecycle_run_status = await this.config_fs.get_config_data(latest_lifecycle_run_status_path, options);
488-
return latest_lifecycle_run_status;
489-
} catch (err) {
490-
if (err.code === 'ENOENT' && silent_if_missing) {
491-
return;
492-
}
493-
throw err;
494-
}
495-
}
496-
497476
/**
498477
* get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred
499478
* @param {string} type

‎src/manage_nsfs/manage_nsfs_constants.js

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ const VALID_OPTIONS_CONNECTION = {
9696
'status': new Set(['name', 'decrypt', ...CLI_MUTUAL_OPTIONS]),
9797
};
9898

99-
const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', ...CLI_MUTUAL_OPTIONS]);
99+
const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', 'continue', ...CLI_MUTUAL_OPTIONS]);
100100

101101
const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]);
102102

@@ -159,7 +159,8 @@ const OPTION_TYPE = {
159159
// lifecycle options
160160
disable_service_validation: 'boolean',
161161
disable_runtime_validation: 'boolean',
162-
short: 'boolean',
162+
short_status: 'boolean',
163+
continue: 'boolean',
163164
//connection
164165
notification_protocol: 'string',
165166
agent_request_object: 'string',
@@ -175,7 +176,7 @@ const OPTION_TYPE = {
175176
const BOOLEAN_STRING_VALUES = ['true', 'false'];
176177
const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
177178
'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous',
178-
'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification']);
179+
'disable_service_validation', 'disable_runtime_validation', 'short_status', 'skip_verification', 'continue']);
179180

180181
// CLI UNSET VALUES
181182
const CLI_EMPTY_STRING = '';

‎src/manage_nsfs/nc_lifecycle.js

Lines changed: 69 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ const { throw_cli_error, get_service_status, NOOBAA_SERVICE_NAME,
1818
is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils');
1919
const SensitiveString = require('../util/sensitive_string');
2020
const {CONFIG_TYPES} = require('../sdk/config_fs');
21+
const lifecycle_utils = require('../util/lifecycle_utils');
2122

2223
// TODO:
2324
// implement
@@ -53,11 +54,12 @@ const TIMED_OPS = Object.freeze({
5354
* run_lifecycle_under_lock runs the lifecycle workflow under a file system lock
5455
* lifecycle workflow is being locked to prevent multiple instances from running the lifecycle workflow
5556
* @param {import('../sdk/config_fs').ConfigFS} config_fs
56-
* @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean}} flags
57+
* @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean, should_continue_last_run?: boolean}} flags
5758
* @returns {Promise<{should_run: Boolean, lifecycle_run_status: Object}>}
5859
*/
5960
async function run_lifecycle_under_lock(config_fs, flags) {
60-
const { disable_service_validation = false, disable_runtime_validation = false, short_status = false } = flags;
61+
const { disable_service_validation = false, disable_runtime_validation = false, short_status = false,
62+
should_continue_last_run = false } = flags;
6163
return_short_status = short_status;
6264
const fs_context = config_fs.fs_context;
6365
const lifecyle_logs_dir_path = config.NC_LIFECYCLE_LOGS_DIR;
@@ -76,7 +78,7 @@ async function run_lifecycle_under_lock(config_fs, flags) {
7678
try {
7779
dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle');
7880
new NoobaaEvent(NoobaaEvent.LIFECYCLE_STARTED).create_event();
79-
await run_lifecycle_or_timeout(config_fs, disable_service_validation);
81+
await run_lifecycle_or_timeout(config_fs, disable_service_validation, should_continue_last_run);
8082
} catch (err) {
8183
dbg.error('run_lifecycle_under_lock failed with error', err, err.code, err.message);
8284
throw err;
@@ -96,13 +98,13 @@ async function run_lifecycle_under_lock(config_fs, flags) {
9698
* @param {boolean} disable_service_validation
9799
* @returns {Promise<Void>}
98100
*/
99-
async function run_lifecycle_or_timeout(config_fs, disable_service_validation) {
101+
async function run_lifecycle_or_timeout(config_fs, disable_service_validation, should_continue_last_run) {
100102
await _call_op_and_update_status({
101103
op_name: TIMED_OPS.RUN_LIFECYLE,
102104
op_func: async () => {
103105
await P.timeout(
104106
config.NC_LIFECYCLE_TIMEOUT_MS,
105-
run_lifecycle(config_fs, disable_service_validation),
107+
run_lifecycle(config_fs, disable_service_validation, should_continue_last_run),
106108
() => ManageCLIError.LifecycleWorkerReachedTimeout
107109
);
108110
}
@@ -115,7 +117,7 @@ async function run_lifecycle_or_timeout(config_fs, disable_service_validation) {
115117
* @param {boolean} disable_service_validation
116118
* @returns {Promise<Void>}
117119
*/
118-
async function run_lifecycle(config_fs, disable_service_validation) {
120+
async function run_lifecycle(config_fs, disable_service_validation, should_continue_last_run) {
119121
const system_json = await config_fs.get_system_config_file(config_fs_options);
120122
if (!disable_service_validation) await throw_if_noobaa_not_active(config_fs, system_json);
121123

@@ -124,6 +126,10 @@ async function run_lifecycle(config_fs, disable_service_validation) {
124126
op_func: async () => config_fs.list_buckets()
125127
});
126128

129+
if (should_continue_last_run) {
130+
await load_previous_run_state(bucket_names, config_fs);
131+
}
132+
127133
await _call_op_and_update_status({
128134
op_name: TIMED_OPS.PROCESS_BUCKETS,
129135
op_func: async () => process_buckets(config_fs, bucket_names, system_json)
@@ -185,7 +191,6 @@ async function process_bucket(config_fs, bucket_name, system_json) {
185191
*/
186192
async function process_rules(config_fs, bucket_json, object_sdk, should_notify) {
187193
try {
188-
lifecycle_run_status.buckets_statuses[bucket_json.name].state ??= {};
189194
const bucket_state = lifecycle_run_status.buckets_statuses[bucket_json.name].state;
190195
bucket_state.num_processed_objects = 0;
191196
while (!bucket_state.is_finished && bucket_state.num_processed_objects < config.NC_LIFECYCLE_BUCKET_BATCH_SIZE) {
@@ -409,7 +414,7 @@ async function throw_if_noobaa_not_active(config_fs, system_json) {
409414
*/
410415
async function get_candidates(bucket_json, lifecycle_rule, object_sdk, fs_context) {
411416
const candidates = { abort_mpu_candidates: [], delete_candidates: [] };
412-
const rule_state = lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id]?.state || {};
417+
const rule_state = lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state;
413418
if (lifecycle_rule.expiration) {
414419
candidates.delete_candidates = await get_candidates_by_expiration_rule(lifecycle_rule, bucket_json, object_sdk, rule_state);
415420
if (lifecycle_rule.expiration.days || lifecycle_rule.expiration.expired_object_delete_marker) {
@@ -755,9 +760,9 @@ function update_status({ bucket_name, rule_id, op_name, op_times, reply = [], er
755760
// TODO - check errors
756761
if (op_times.start_time) {
757762
if (op_name === TIMED_OPS.PROCESS_RULE) {
758-
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] = { rule_process_times: {}, rule_stats: {} };
763+
init_rule_status(bucket_name, rule_id);
759764
} else if (op_name === TIMED_OPS.PROCESS_BUCKET) {
760-
lifecycle_run_status.buckets_statuses[bucket_name] = { bucket_process_times: {}, bucket_stats: {}, rules_statuses: {} };
765+
init_bucket_status(bucket_name);
761766
}
762767
}
763768
_update_times_on_status({ op_name, op_times, bucket_name, rule_id });
@@ -882,6 +887,60 @@ async function write_lifecycle_log_file(fs_context, lifecyle_logs_dir_path) {
882887
);
883888
}
884889

890+
/**
891+
* init the bucket status object statuses if they don't exist
892+
* @param {string} bucket_name
893+
* @returns {Object} created or existing bucket status
894+
*/
895+
function init_bucket_status(bucket_name) {
896+
lifecycle_run_status.buckets_statuses[bucket_name] ??= {};
897+
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses ??= {};
898+
lifecycle_run_status.buckets_statuses[bucket_name].state ??= {};
899+
lifecycle_run_status.buckets_statuses[bucket_name].bucket_process_times = {};
900+
lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats = {};
901+
return lifecycle_run_status.buckets_statuses[bucket_name];
902+
}
903+
904+
/**
905+
* init the rule status object statuses if they don't exist
906+
* @param {string} bucket_name
907+
* @param {string} rule_id
908+
* @returns {Object} created or existing rule status
909+
*/
910+
function init_rule_status(bucket_name, rule_id) {
911+
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] ??= {};
912+
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].state ??= {};
913+
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times = {};
914+
lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats = {};
915+
return lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id];
916+
}
917+
918+
/**
919+
*
920+
* @param {Object[]} buckets
921+
* @param {Object} config_fs
922+
* @returns
923+
*/
924+
async function load_previous_run_state(buckets, config_fs) {
925+
const previous_run = await lifecycle_utils.get_latest_nc_lifecycle_run_status(config_fs, { silent_if_missing: true });
926+
if (previous_run) {
927+
lifecycle_run_status.state = previous_run.state;
928+
for (const [bucket_name, prev_bucket_status] of Object.entries(previous_run.buckets_statuses)) {
929+
if (!buckets.includes(bucket_name)) continue;
930+
const bucket_json = await config_fs.get_bucket_by_name(bucket_name, config_fs_options);
931+
if (!bucket_json.lifecycle_configuration_rules) continue;
932+
const bucket_status = init_bucket_status(bucket_name);
933+
bucket_status.state = prev_bucket_status.state;
934+
const bucket_rules = bucket_json.lifecycle_configuration_rules.map(rule => rule.id);
935+
for (const [rule_id, prev_rule_status] of Object.entries(prev_bucket_status.rules_statuses)) {
936+
if (!bucket_rules.includes(rule_id)) return;
937+
const rule_status = init_rule_status(bucket_name, rule_id);
938+
rule_status.state = prev_rule_status.state;
939+
}
940+
}
941+
}
942+
}
943+
885944
// EXPORTS
886945
exports.run_lifecycle_under_lock = run_lifecycle_under_lock;
887946

‎src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js

Lines changed: 94 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -481,13 +481,14 @@ describe('noobaa cli - lifecycle batching', () => {
481481
object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined);
482482
object_sdk.requesting_account = json_account;
483483
await object_sdk.create_bucket({ name: test_bucket });
484-
config.NC_LIFECYCLE_LOGS_DIR = tmp_lifecycle_logs_dir_path;
485-
config.NC_LIFECYCLE_LIST_BATCH_SIZE = 2;
486-
config.NC_LIFECYCLE_BUCKET_BATCH_SIZE = 5;
487484
});
488485

489486
beforeEach(async () => {
490-
await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path }));
487+
await config_fs.create_config_json_file(JSON.stringify({
488+
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
489+
NC_LIFECYCLE_LIST_BATCH_SIZE: 2,
490+
NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
491+
}));
491492
});
492493

493494
afterEach(async () => {
@@ -583,7 +584,9 @@ describe('noobaa cli - lifecycle batching', () => {
583584

584585
await config_fs.update_config_json_file(JSON.stringify({
585586
NC_LIFECYCLE_TIMEOUT_MS: 1,
586-
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path
587+
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
588+
NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
589+
NC_LIFECYCLE_LIST_BATCH_SIZE: 2
587590
}));
588591
await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
589592

@@ -595,6 +598,92 @@ describe('noobaa cli - lifecycle batching', () => {
595598
const object_list = await object_sdk.list_objects({bucket: test_bucket});
596599
expect(object_list.objects.length).not.toBe(0);
597600
});
601+
602+
it("lifecycle batching - continue finished lifecycle should do nothing", async () => {
603+
await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
604+
605+
await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
606+
await create_object(object_sdk, test_bucket, test_key1, 100, true);
607+
await create_object(object_sdk, test_bucket, test_key2, 100, true);
608+
609+
//continue finished run
610+
await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined);
611+
612+
const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
613+
expect(lifecycle_log_entries.length).toBe(2);
614+
const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name);
615+
const lifecycle_log_json = await config_fs.get_config_data(log_file_path, {silent_if_missing: true});
616+
expect(lifecycle_log_json.state.is_finished).toBe(true);
617+
const object_list = await object_sdk.list_objects({bucket: test_bucket});
618+
expect(object_list.objects.length).toBe(2);
619+
});
620+
621+
it("continue lifecycle batching should finish the run - delete all", async () => {
622+
await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
623+
await config_fs.update_config_json_file(JSON.stringify({
624+
NC_LIFECYCLE_TIMEOUT_MS: 60,
625+
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
626+
NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5,
627+
NC_LIFECYCLE_LIST_BATCH_SIZE: 2
628+
629+
}));
630+
const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"];
631+
for (const key of keys) {
632+
await create_object(object_sdk, test_bucket, key, 100, false);
633+
}
634+
try {
635+
await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
636+
} catch (e) {
637+
//ignore error
638+
}
639+
await exec_manage_cli(TYPES.LIFECYCLE, '', {continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
640+
const object_list2 = await object_sdk.list_objects({bucket: test_bucket});
641+
expect(object_list2.objects.length).toBe(0);
642+
});
643+
644+
it("continue lifecycle batching should finish the run - validate new run. don't delete already deleted items", async () => {
645+
await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all });
646+
await config_fs.update_config_json_file(JSON.stringify({
647+
NC_LIFECYCLE_TIMEOUT_MS: 70,
648+
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
649+
NC_LIFECYCLE_BUCKET_BATCH_SIZE: 4,
650+
NC_LIFECYCLE_LIST_BATCH_SIZE: 2
651+
}));
652+
const keys = [];
653+
for (let i = 0; i < 100; i++) {
654+
const new_key = `key${i}`;
655+
await create_object(object_sdk, test_bucket, new_key, 100, false);
656+
keys.push(new_key);
657+
}
658+
try {
659+
await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
660+
} catch (e) {
661+
//ignore error
662+
}
663+
664+
const object_list = await object_sdk.list_objects({bucket: test_bucket});
665+
const intermediate_key_list = object_list.objects.map(object => object.key);
666+
const new_keys = [];
667+
//recreate deleted key. next run should skip those keys
668+
for (const key of keys) {
669+
if (!intermediate_key_list.includes(key)) {
670+
await create_object(object_sdk, test_bucket, key, 100, false);
671+
new_keys.push(key);
672+
}
673+
}
674+
await config_fs.update_config_json_file(JSON.stringify({
675+
NC_LIFECYCLE_TIMEOUT_MS: 9999,
676+
NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path,
677+
NC_LIFECYCLE_BUCKET_BATCH_SIZE: 4,
678+
NC_LIFECYCLE_LIST_BATCH_SIZE: 2,
679+
}));
680+
await exec_manage_cli(TYPES.LIFECYCLE, '', {continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
681+
const object_list2 = await object_sdk.list_objects({bucket: test_bucket});
682+
const res_keys = object_list2.objects.map(object => object.key);
683+
for (const key of new_keys) {
684+
expect(res_keys).toContain(key);
685+
}
686+
});
598687
});
599688

600689
describe('noobaa cli - lifecycle notifications', () => {

0 commit comments

Comments
 (0)