1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
3
4 #include "ImageSync.h"
5 #include "InstanceWatcher.h"
6 #include "ProgressContext.h"
7 #include "common/debug.h"
8 #include "common/Timer.h"
9 #include "common/errno.h"
10 #include "journal/Journaler.h"
11 #include "librbd/DeepCopyRequest.h"
12 #include "librbd/ImageCtx.h"
13 #include "librbd/ImageState.h"
14 #include "librbd/Utils.h"
15 #include "librbd/internal.h"
16 #include "librbd/journal/Types.h"
17 #include "tools/rbd_mirror/image_sync/SyncPointCreateRequest.h"
18 #include "tools/rbd_mirror/image_sync/SyncPointPruneRequest.h"
19
20 #define dout_context g_ceph_context
21 #define dout_subsys ceph_subsys_rbd_mirror
22 #undef dout_prefix
23 #define dout_prefix *_dout << "rbd::mirror::ImageSync: " \
24 << this << " " << __func__
25
26 namespace rbd {
27 namespace mirror {
28
29 using namespace image_sync;
30 using librbd::util::create_async_context_callback;
31 using librbd::util::create_context_callback;
32 using librbd::util::unique_lock_name;
33
34 template <typename I>
35 class ImageSync<I>::ImageCopyProgressContext : public librbd::ProgressContext {
36 public:
37 ImageCopyProgressContext(ImageSync *image_sync) : image_sync(image_sync) {
38 }
39
40 int update_progress(uint64_t object_no, uint64_t object_count) override {
41 image_sync->handle_copy_image_update_progress(object_no, object_count);
42 return 0;
43 }
44
45 ImageSync *image_sync;
46 };
47
48 template <typename I>
49 ImageSync<I>::ImageSync(I *local_image_ctx, I *remote_image_ctx,
50 SafeTimer *timer, ceph::mutex *timer_lock,
51 const std::string &mirror_uuid, Journaler *journaler,
52 MirrorPeerClientMeta *client_meta,
53 ContextWQ *work_queue,
54 InstanceWatcher<I> *instance_watcher,
55 Context *on_finish, ProgressContext *progress_ctx)
56 : BaseRequest("rbd::mirror::ImageSync", local_image_ctx->cct, on_finish),
57 m_local_image_ctx(local_image_ctx), m_remote_image_ctx(remote_image_ctx),
58 m_timer(timer), m_timer_lock(timer_lock), m_mirror_uuid(mirror_uuid),
59 m_journaler(journaler), m_client_meta(client_meta),
60 m_work_queue(work_queue), m_instance_watcher(instance_watcher),
61 m_progress_ctx(progress_ctx),
62 m_lock(ceph::make_mutex(unique_lock_name("ImageSync::m_lock", this))),
63 m_update_sync_point_interval(m_local_image_ctx->cct->_conf.template get_val<double>(
64 "rbd_mirror_sync_point_update_age")), m_client_meta_copy(*client_meta) {
65 }
66
67 template <typename I>
|
(1) Event exn_spec_violation: |
An exception of type "_ZN5boost16exception_detail10clone_implINS0_19error_info_injectorINSt8ios_base7failureB5cxx11EEEEE" is thrown but the throw list "throw()" doesn't allow it to be thrown. This will cause a call to unexpected() which usually calls terminate(). |
| Also see events: |
[fun_call_w_exception] |
68 ImageSync<I>::~ImageSync() {
69 ceph_assert(m_image_copy_request == nullptr);
|
(2) Event fun_call_w_exception: |
Called function throws an exception of type "_ZN5boost16exception_detail10clone_implINS0_19error_info_injectorINSt8ios_base7failureB5cxx11EEEEE". [details] |
| Also see events: |
[exn_spec_violation] |
70 ceph_assert(m_image_copy_prog_ctx == nullptr);
71 ceph_assert(m_update_sync_ctx == nullptr);
72 }
73
74 template <typename I>
75 void ImageSync<I>::send() {
76 send_notify_sync_request();
77 }
78
79 template <typename I>
80 void ImageSync<I>::cancel() {
81 std::lock_guard locker{m_lock};
82
83 dout(10) << dendl;
84
85 m_canceled = true;
86
87 if (m_instance_watcher->cancel_sync_request(m_local_image_ctx->id)) {
88 return;
89 }
90
91 if (m_image_copy_request != nullptr) {
92 m_image_copy_request->cancel();
93 }
94 }
95
96 template <typename I>
97 void ImageSync<I>::send_notify_sync_request() {
98 update_progress("NOTIFY_SYNC_REQUEST");
99
100 dout(10) << dendl;
101
102 m_lock.lock();
103 if (m_canceled) {
104 m_lock.unlock();
105 BaseRequest::finish(-ECANCELED);
106 return;
107 }
108
109 Context *ctx = create_async_context_callback(
110 m_work_queue, create_context_callback<
111 ImageSync<I>, &ImageSync<I>::handle_notify_sync_request>(this));
112 m_instance_watcher->notify_sync_request(m_local_image_ctx->id, ctx);
113 m_lock.unlock();
114 }
115
116 template <typename I>
117 void ImageSync<I>::handle_notify_sync_request(int r) {
118 dout(10) << ": r=" << r << dendl;
119
120 m_lock.lock();
121 if (r == 0 && m_canceled) {
122 r = -ECANCELED;
123 }
124 m_lock.unlock();
125
126 if (r < 0) {
127 BaseRequest::finish(r);
128 return;
129 }
130
131 send_prune_catch_up_sync_point();
132 }
133
134 template <typename I>
135 void ImageSync<I>::send_prune_catch_up_sync_point() {
136 update_progress("PRUNE_CATCH_UP_SYNC_POINT");
137
138 if (m_client_meta->sync_points.empty()) {
139 send_create_sync_point();
140 return;
141 }
142
143 dout(10) << dendl;
144
145 // prune will remove sync points with missing snapshots and
146 // ensure we have a maximum of one sync point (in case we
147 // restarted)
148 Context *ctx = create_context_callback<
149 ImageSync<I>, &ImageSync<I>::handle_prune_catch_up_sync_point>(this);
150 SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create(
151 m_remote_image_ctx, false, m_journaler, m_client_meta, ctx);
152 request->send();
153 }
154
155 template <typename I>
156 void ImageSync<I>::handle_prune_catch_up_sync_point(int r) {
157 dout(10) << ": r=" << r << dendl;
158
159 if (r < 0) {
160 derr << ": failed to prune catch-up sync point: "
161 << cpp_strerror(r) << dendl;
162 finish(r);
163 return;
164 }
165
166 send_create_sync_point();
167 }
168
169 template <typename I>
170 void ImageSync<I>::send_create_sync_point() {
171 update_progress("CREATE_SYNC_POINT");
172
173 // TODO: when support for disconnecting laggy clients is added,
174 // re-connect and create catch-up sync point
175 if (m_client_meta->sync_points.size() > 0) {
176 send_copy_image();
177 return;
178 }
179
180 dout(10) << dendl;
181
182 Context *ctx = create_context_callback<
183 ImageSync<I>, &ImageSync<I>::handle_create_sync_point>(this);
184 SyncPointCreateRequest<I> *request = SyncPointCreateRequest<I>::create(
185 m_remote_image_ctx, m_mirror_uuid, m_journaler, m_client_meta, ctx);
186 request->send();
187 }
188
189 template <typename I>
190 void ImageSync<I>::handle_create_sync_point(int r) {
191 dout(10) << ": r=" << r << dendl;
192
193 if (r < 0) {
194 derr << ": failed to create sync point: " << cpp_strerror(r)
195 << dendl;
196 finish(r);
197 return;
198 }
199
200 send_copy_image();
201 }
202
203 template <typename I>
204 void ImageSync<I>::send_copy_image() {
205 librados::snap_t snap_id_start = 0;
206 librados::snap_t snap_id_end;
207 librbd::deep_copy::ObjectNumber object_number;
208 int r = 0;
209 {
210 std::shared_lock image_locker{m_remote_image_ctx->image_lock};
211 ceph_assert(!m_client_meta->sync_points.empty());
212 auto &sync_point = m_client_meta->sync_points.front();
213 snap_id_end = m_remote_image_ctx->get_snap_id(
214 cls::rbd::UserSnapshotNamespace(), sync_point.snap_name);
215 if (snap_id_end == CEPH_NOSNAP) {
216 derr << ": failed to locate snapshot: " << sync_point.snap_name << dendl;
217 r = -ENOENT;
218 } else if (!sync_point.from_snap_name.empty()) {
219 snap_id_start = m_remote_image_ctx->get_snap_id(
220 cls::rbd::UserSnapshotNamespace(), sync_point.from_snap_name);
221 if (snap_id_start == CEPH_NOSNAP) {
222 derr << ": failed to locate from snapshot: "
223 << sync_point.from_snap_name << dendl;
224 r = -ENOENT;
225 }
226 }
227 object_number = sync_point.object_number;
228 }
229 if (r < 0) {
230 finish(r);
231 return;
232 }
233
234 m_lock.lock();
235 if (m_canceled) {
236 m_lock.unlock();
237 finish(-ECANCELED);
238 return;
239 }
240
241 dout(10) << dendl;
242
243 Context *ctx = create_context_callback<
244 ImageSync<I>, &ImageSync<I>::handle_copy_image>(this);
245 m_image_copy_prog_ctx = new ImageCopyProgressContext(this);
246 m_image_copy_request = librbd::DeepCopyRequest<I>::create(
247 m_remote_image_ctx, m_local_image_ctx, snap_id_start, snap_id_end,
248 false, object_number, m_work_queue, &m_client_meta->snap_seqs,
249 m_image_copy_prog_ctx, ctx);
250 m_image_copy_request->get();
251 m_lock.unlock();
252
253 update_progress("COPY_IMAGE");
254
255 m_image_copy_request->send();
256 }
257
258 template <typename I>
259 void ImageSync<I>::handle_copy_image(int r) {
260 dout(10) << ": r=" << r << dendl;
261
262 {
263 std::scoped_lock locker{*m_timer_lock, m_lock};
264 m_image_copy_request->put();
265 m_image_copy_request = nullptr;
266 delete m_image_copy_prog_ctx;
267 m_image_copy_prog_ctx = nullptr;
268 if (r == 0 && m_canceled) {
269 r = -ECANCELED;
270 }
271
272 if (m_update_sync_ctx != nullptr) {
273 m_timer->cancel_event(m_update_sync_ctx);
274 m_update_sync_ctx = nullptr;
275 }
276
277 if (m_updating_sync_point) {
278 m_ret_val = r;
279 return;
280 }
281 }
282
283 if (r == -ECANCELED) {
284 dout(10) << ": image copy canceled" << dendl;
285 finish(r);
286 return;
287 } else if (r < 0) {
288 derr << ": failed to copy image: " << cpp_strerror(r) << dendl;
289 finish(r);
290 return;
291 }
292
293 send_flush_sync_point();
294 }
295
296 template <typename I>
297 void ImageSync<I>::handle_copy_image_update_progress(uint64_t object_no,
298 uint64_t object_count) {
299 int percent = 100 * object_no / object_count;
300 update_progress("COPY_IMAGE " + stringify(percent) + "%");
301
302 std::lock_guard locker{m_lock};
303 m_image_copy_object_no = object_no;
304 m_image_copy_object_count = object_count;
305
306 if (m_update_sync_ctx == nullptr && !m_updating_sync_point) {
307 send_update_sync_point();
308 }
309 }
310
311 template <typename I>
312 void ImageSync<I>::send_update_sync_point() {
313 ceph_assert(ceph_mutex_is_locked(m_lock));
314
315 m_update_sync_ctx = nullptr;
316
317 if (m_canceled) {
318 return;
319 }
320
321 auto sync_point = &m_client_meta->sync_points.front();
322
323 if (m_client_meta->sync_object_count == m_image_copy_object_count &&
324 sync_point->object_number &&
325 (m_image_copy_object_no - 1) == sync_point->object_number.get()) {
326 // update sync point did not progress since last sync
327 return;
328 }
329
330 m_updating_sync_point = true;
331
332 m_client_meta_copy = *m_client_meta;
333 m_client_meta->sync_object_count = m_image_copy_object_count;
334 if (m_image_copy_object_no > 0) {
335 sync_point->object_number = m_image_copy_object_no - 1;
336 }
337
338 CephContext *cct = m_local_image_ctx->cct;
339 ldout(cct, 20) << ": sync_point=" << *sync_point << dendl;
340
341 bufferlist client_data_bl;
342 librbd::journal::ClientData client_data(*m_client_meta);
343 encode(client_data, client_data_bl);
344
345 Context *ctx = create_context_callback<
346 ImageSync<I>, &ImageSync<I>::handle_update_sync_point>(
347 this);
348 m_journaler->update_client(client_data_bl, ctx);
349 }
350
351 template <typename I>
352 void ImageSync<I>::handle_update_sync_point(int r) {
353 CephContext *cct = m_local_image_ctx->cct;
354 ldout(cct, 20) << ": r=" << r << dendl;
355
356 if (r < 0) {
357 *m_client_meta = m_client_meta_copy;
358 lderr(cct) << ": failed to update client data: " << cpp_strerror(r)
359 << dendl;
360 }
361
362 {
363 std::scoped_lock locker{*m_timer_lock, m_lock};
364 m_updating_sync_point = false;
365
366 if (m_image_copy_request != nullptr) {
367 m_update_sync_ctx = new LambdaContext(
368 [this](int r) {
369 std::lock_guard locker{m_lock};
370 this->send_update_sync_point();
371 });
372 m_timer->add_event_after(m_update_sync_point_interval,
373 m_update_sync_ctx);
374 return;
375 }
376 }
377
378 send_flush_sync_point();
379 }
380
381 template <typename I>
382 void ImageSync<I>::send_flush_sync_point() {
383 if (m_ret_val < 0) {
384 finish(m_ret_val);
385 return;
386 }
387
388 update_progress("FLUSH_SYNC_POINT");
389
390 m_client_meta_copy = *m_client_meta;
391 m_client_meta->sync_object_count = m_image_copy_object_count;
392 auto sync_point = &m_client_meta->sync_points.front();
393 if (m_image_copy_object_no > 0) {
394 sync_point->object_number = m_image_copy_object_no - 1;
395 } else {
396 sync_point->object_number = boost::none;
397 }
398
399 dout(10) << ": sync_point=" << *sync_point << dendl;
400
401 bufferlist client_data_bl;
402 librbd::journal::ClientData client_data(*m_client_meta);
403 encode(client_data, client_data_bl);
404
405 Context *ctx = create_context_callback<
406 ImageSync<I>, &ImageSync<I>::handle_flush_sync_point>(
407 this);
408 m_journaler->update_client(client_data_bl, ctx);
409 }
410
411 template <typename I>
412 void ImageSync<I>::handle_flush_sync_point(int r) {
413 dout(10) << ": r=" << r << dendl;
414
415 if (r < 0) {
416 *m_client_meta = m_client_meta_copy;
417
418 derr << ": failed to update client data: " << cpp_strerror(r)
419 << dendl;
420 finish(r);
421 return;
422 }
423
424 send_prune_sync_points();
425 }
426
427 template <typename I>
428 void ImageSync<I>::send_prune_sync_points() {
429 dout(10) << dendl;
430
431 update_progress("PRUNE_SYNC_POINTS");
432
433 Context *ctx = create_context_callback<
434 ImageSync<I>, &ImageSync<I>::handle_prune_sync_points>(this);
435 SyncPointPruneRequest<I> *request = SyncPointPruneRequest<I>::create(
436 m_remote_image_ctx, true, m_journaler, m_client_meta, ctx);
437 request->send();
438 }
439
440 template <typename I>
441 void ImageSync<I>::handle_prune_sync_points(int r) {
442 dout(10) << ": r=" << r << dendl;
443
444 if (r < 0) {
445 derr << ": failed to prune sync point: "
446 << cpp_strerror(r) << dendl;
447 finish(r);
448 return;
449 }
450
451 if (!m_client_meta->sync_points.empty()) {
452 send_copy_image();
453 return;
454 }
455
456 finish(0);
457 }
458
459 template <typename I>
460 void ImageSync<I>::update_progress(const std::string &description) {
461 dout(20) << ": " << description << dendl;
462
463 if (m_progress_ctx) {
464 m_progress_ctx->update_progress("IMAGE_SYNC/" + description);
465 }
466 }
467
468 template <typename I>
469 void ImageSync<I>::finish(int r) {
470 dout(20) << ": r=" << r << dendl;
471
472 m_instance_watcher->notify_sync_complete(m_local_image_ctx->id);
473 BaseRequest::finish(r);
474 }
475
476 } // namespace mirror
477 } // namespace rbd
478
479 template class rbd::mirror::ImageSync<librbd::ImageCtx>;
480