1
// This file is part of Substrate.
2

            
3
// Copyright (C) Parity Technologies (UK) Ltd.
4
// SPDX-License-Identifier: Apache-2.0
5

            
6
// Licensed under the Apache License, Version 2.0 (the "License");
7
// you may not use this file except in compliance with the License.
8
// You may obtain a copy of the License at
9
//
10
// 	http://www.apache.org/licenses/LICENSE-2.0
11
//
12
// Unless required by applicable law or agreed to in writing, software
13
// distributed under the License is distributed on an "AS IS" BASIS,
14
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
// See the License for the specific language governing permissions and
16
// limitations under the License.
17

            
18
//! Migrations for the scheduler pallet.
19

            
20
use super::*;
21
use frame_support::traits::OnRuntimeUpgrade;
22
use frame_system::pallet_prelude::BlockNumberFor;
23

            
24
#[cfg(feature = "try-runtime")]
25
use sp_runtime::TryRuntimeError;
26

            
27
/// The log target.
28
const TARGET: &'static str = "runtime::scheduler::migration";
29

            
30
pub mod v1 {
31
	use super::*;
32
	use frame_support::pallet_prelude::*;
33

            
34
	#[frame_support::storage_alias]
35
	pub(crate) type Agenda<T: Config> = StorageMap<
36
		Pallet<T>,
37
		Twox64Concat,
38
		BlockNumberFor<T>,
39
		Vec<Option<ScheduledV1<<T as Config>::RuntimeCall, BlockNumberFor<T>>>>,
40
		ValueQuery,
41
	>;
42

            
43
	#[frame_support::storage_alias]
44
	pub(crate) type Lookup<T: Config> =
45
		StorageMap<Pallet<T>, Twox64Concat, Vec<u8>, TaskAddress<BlockNumberFor<T>>>;
46
}
47

            
48
pub mod v2 {
49
	use super::*;
50
	use frame_support::pallet_prelude::*;
51

            
52
	#[frame_support::storage_alias]
53
	pub(crate) type Agenda<T: Config> = StorageMap<
54
		Pallet<T>,
55
		Twox64Concat,
56
		BlockNumberFor<T>,
57
		Vec<Option<ScheduledV2Of<T>>>,
58
		ValueQuery,
59
	>;
60

            
61
	#[frame_support::storage_alias]
62
	pub(crate) type Lookup<T: Config> =
63
		StorageMap<Pallet<T>, Twox64Concat, Vec<u8>, TaskAddress<BlockNumberFor<T>>>;
64
}
65

            
66
pub mod v3 {
67
	use super::*;
68
	use frame_support::pallet_prelude::*;
69

            
70
	#[frame_support::storage_alias]
71
	pub(crate) type Agenda<T: Config> = StorageMap<
72
		Pallet<T>,
73
		Twox64Concat,
74
		BlockNumberFor<T>,
75
		Vec<Option<ScheduledV3Of<T>>>,
76
		ValueQuery,
77
	>;
78

            
79
	#[frame_support::storage_alias]
80
	pub(crate) type Lookup<T: Config> =
81
		StorageMap<Pallet<T>, Twox64Concat, Vec<u8>, TaskAddress<BlockNumberFor<T>>>;
82

            
83
	/// Migrate the scheduler pallet from V3 to V4.
84
	pub struct MigrateToV4<T>(core::marker::PhantomData<T>);
85

            
86
	impl<T: Config> OnRuntimeUpgrade for MigrateToV4<T> {
87
		#[cfg(feature = "try-runtime")]
88
		fn pre_upgrade() -> Result<Vec<u8>, TryRuntimeError> {
89
			ensure!(StorageVersion::get::<Pallet<T>>() == 3, "Can only upgrade from version 3");
90

            
91
			let agendas = Agenda::<T>::iter_keys().count() as u32;
92
			let decodable_agendas = Agenda::<T>::iter_values().count() as u32;
93
			if agendas != decodable_agendas {
94
				// This is not necessarily an error, but can happen when there are Calls
95
				// in an Agenda that are not valid anymore with the new runtime.
96
				log::error!(
97
					target: TARGET,
98
					"Can only decode {} of {} agendas - others will be dropped",
99
					decodable_agendas,
100
					agendas
101
				);
102
			}
103
			log::info!(target: TARGET, "Trying to migrate {} agendas...", decodable_agendas);
104

            
105
			// Check that no agenda overflows `MaxScheduledPerBlock`.
106
			let max_scheduled_per_block = T::MaxScheduledPerBlock::get() as usize;
107
			for (block_number, agenda) in Agenda::<T>::iter() {
108
				if agenda.iter().cloned().flatten().count() > max_scheduled_per_block {
109
					log::error!(
110
						target: TARGET,
111
						"Would truncate agenda of block {:?} from {} items to {} items.",
112
						block_number,
113
						agenda.len(),
114
						max_scheduled_per_block,
115
					);
116
					return Err("Agenda would overflow `MaxScheduledPerBlock`.".into())
117
				}
118
			}
119
			// Check that bounding the calls will not overflow `MAX_LENGTH`.
120
			let max_length = T::Preimages::MAX_LENGTH as usize;
121
			for (block_number, agenda) in Agenda::<T>::iter() {
122
				for schedule in agenda.iter().cloned().flatten() {
123
					match schedule.call {
124
						frame_support::traits::schedule::MaybeHashed::Value(call) => {
125
							let l = call.using_encoded(|c| c.len());
126
							if l > max_length {
127
								log::error!(
128
									target: TARGET,
129
									"Call in agenda of block {:?} is too large: {} byte",
130
									block_number,
131
									l,
132
								);
133
								return Err("Call is too large.".into())
134
							}
135
						},
136
						_ => (),
137
					}
138
				}
139
			}
140

            
141
			Ok((decodable_agendas as u32).encode())
142
		}
143

            
144
		fn on_runtime_upgrade() -> Weight {
145
			let version = StorageVersion::get::<Pallet<T>>();
146
			if version != 3 {
147
				log::warn!(
148
					target: TARGET,
149
					"skipping v3 to v4 migration: executed on wrong storage version.\
150
				Expected version 3, found {:?}",
151
					version,
152
				);
153
				return T::DbWeight::get().reads(1)
154
			}
155

            
156
			crate::Pallet::<T>::migrate_v3_to_v4()
157
		}
158

            
159
		#[cfg(feature = "try-runtime")]
160
		fn post_upgrade(state: Vec<u8>) -> Result<(), TryRuntimeError> {
161
			ensure!(StorageVersion::get::<Pallet<T>>() == 4, "Must upgrade");
162

            
163
			// Check that everything decoded fine.
164
			for k in crate::Agenda::<T>::iter_keys() {
165
				ensure!(crate::Agenda::<T>::try_get(k).is_ok(), "Cannot decode V4 Agenda");
166
			}
167

            
168
			let old_agendas: u32 =
169
				Decode::decode(&mut &state[..]).expect("pre_upgrade provides a valid state; qed");
170
			let new_agendas = crate::Agenda::<T>::iter_keys().count() as u32;
171
			if old_agendas != new_agendas {
172
				// This is not necessarily an error, but can happen when there are Calls
173
				// in an Agenda that are not valid anymore in the new runtime.
174
				log::error!(
175
					target: TARGET,
176
					"Did not migrate all Agendas. Previous {}, Now {}",
177
					old_agendas,
178
					new_agendas,
179
				);
180
			} else {
181
				log::info!(target: TARGET, "Migrated {} agendas.", new_agendas);
182
			}
183

            
184
			Ok(())
185
		}
186
	}
187
}
188

            
189
pub mod v4 {
190
	use super::*;
191
	use frame_support::pallet_prelude::*;
192

            
193
	/// This migration cleans up empty agendas of the V4 scheduler.
194
	///
195
	/// This should be run on a scheduler that does not have
196
	/// <https://github.com/paritytech/substrate/pull/12989> since it piles up `None`-only agendas. This does not modify the pallet version.
197
	pub struct CleanupAgendas<T>(core::marker::PhantomData<T>);
198

            
199
	impl<T: Config> OnRuntimeUpgrade for CleanupAgendas<T> {
200
		#[cfg(feature = "try-runtime")]
201
		fn pre_upgrade() -> Result<Vec<u8>, TryRuntimeError> {
202
			assert_eq!(
203
				StorageVersion::get::<Pallet<T>>(),
204
				4,
205
				"Can only cleanup agendas of the V4 scheduler"
206
			);
207

            
208
			let agendas = Agenda::<T>::iter_keys().count();
209
			let non_empty_agendas =
210
				Agenda::<T>::iter_values().filter(|a| a.iter().any(|s| s.is_some())).count();
211
			log::info!(
212
				target: TARGET,
213
				"There are {} total and {} non-empty agendas",
214
				agendas,
215
				non_empty_agendas
216
			);
217

            
218
			Ok((agendas as u32, non_empty_agendas as u32).encode())
219
		}
220

            
221
		fn on_runtime_upgrade() -> Weight {
222
			let version = StorageVersion::get::<Pallet<T>>();
223
			if version != 4 {
224
				log::warn!(target: TARGET, "Skipping CleanupAgendas migration since it was run on the wrong version: {:?} != 4", version);
225
				return T::DbWeight::get().reads(1)
226
			}
227

            
228
			let keys = Agenda::<T>::iter_keys().collect::<Vec<_>>();
229
			let mut writes = 0;
230
			for k in &keys {
231
				let mut schedules = Agenda::<T>::get(k);
232
				let all_schedules = schedules.len();
233
				let suffix_none_schedules =
234
					schedules.iter().rev().take_while(|s| s.is_none()).count();
235

            
236
				match all_schedules.checked_sub(suffix_none_schedules) {
237
					Some(0) => {
238
						log::info!(
239
							"Deleting None-only agenda {:?} with {} entries",
240
							k,
241
							all_schedules
242
						);
243
						Agenda::<T>::remove(k);
244
						writes.saturating_inc();
245
					},
246
					Some(ne) if ne > 0 => {
247
						log::info!(
248
							"Removing {} schedules of {} from agenda {:?}, now {:?}",
249
							suffix_none_schedules,
250
							all_schedules,
251
							ne,
252
							k
253
						);
254
						schedules.truncate(ne);
255
						Agenda::<T>::insert(k, schedules);
256
						writes.saturating_inc();
257
					},
258
					Some(_) => {
259
						frame_support::defensive!(
260
							// Bad but let's not panic.
261
							"Cannot have more None suffix schedules that schedules in total"
262
						);
263
					},
264
					None => {
265
						log::info!("Agenda {:?} does not have any None suffix schedules", k);
266
					},
267
				}
268
			}
269

            
270
			// We don't modify the pallet version.
271

            
272
			T::DbWeight::get().reads_writes(1 + keys.len().saturating_mul(2) as u64, writes)
273
		}
274

            
275
		#[cfg(feature = "try-runtime")]
276
		fn post_upgrade(state: Vec<u8>) -> Result<(), TryRuntimeError> {
277
			ensure!(StorageVersion::get::<Pallet<T>>() == 4, "Version must not change");
278

            
279
			let (old_agendas, non_empty_agendas): (u32, u32) =
280
				Decode::decode(&mut state.as_ref()).expect("Must decode pre_upgrade state");
281
			let new_agendas = Agenda::<T>::iter_keys().count() as u32;
282

            
283
			match old_agendas.checked_sub(new_agendas) {
284
				Some(0) => log::warn!(
285
					target: TARGET,
286
					"Did not clean up any agendas. v4::CleanupAgendas can be removed."
287
				),
288
				Some(n) => {
289
					log::info!(target: TARGET, "Cleaned up {} agendas, now {}", n, new_agendas)
290
				},
291
				None => unreachable!(
292
					"Number of agendas cannot increase, old {} new {}",
293
					old_agendas, new_agendas
294
				),
295
			}
296
			ensure!(new_agendas == non_empty_agendas, "Expected to keep all non-empty agendas");
297

            
298
			Ok(())
299
		}
300
	}
301
}
302

            
303
#[cfg(test)]
304
#[cfg(feature = "try-runtime")]
305
mod test {
306
	use super::*;
307
	use crate::mock::*;
308
	use alloc::borrow::Cow;
309
	use frame_support::Hashable;
310
	use substrate_test_utils::assert_eq_uvec;
311

            
312
	#[test]
313
	#[allow(deprecated)]
314
	fn migration_v3_to_v4_works() {
315
		new_test_ext().execute_with(|| {
316
			// Assume that we are at V3.
317
			StorageVersion::new(3).put::<Scheduler>();
318

            
319
			// Call that will be bounded to a `Lookup`.
320
			let large_call =
321
				RuntimeCall::System(frame_system::Call::remark { remark: vec![0; 1024] });
322
			// Call that can be inlined.
323
			let small_call =
324
				RuntimeCall::System(frame_system::Call::remark { remark: vec![0; 10] });
325
			// Call that is already hashed and can will be converted to `Legacy`.
326
			let hashed_call =
327
				RuntimeCall::System(frame_system::Call::remark { remark: vec![0; 2048] });
328
			let bound_hashed_call = Preimage::bound(hashed_call.clone()).unwrap();
329
			assert!(bound_hashed_call.lookup_needed());
330
			// A Call by hash that will fail to decode becomes `None`.
331
			let trash_data = vec![255u8; 1024];
332
			let undecodable_hash = Preimage::note(Cow::Borrowed(&trash_data)).unwrap();
333

            
334
			for i in 0..2u64 {
335
				let k = i.twox_64_concat();
336
				let old = vec![
337
					Some(ScheduledV3Of::<Test> {
338
						maybe_id: None,
339
						priority: i as u8 + 10,
340
						call: small_call.clone().into(),
341
						maybe_periodic: None, // 1
342
						origin: root(),
343
						_phantom: PhantomData::<u64>::default(),
344
					}),
345
					None,
346
					Some(ScheduledV3Of::<Test> {
347
						maybe_id: Some(vec![i as u8; 32]),
348
						priority: 123,
349
						call: large_call.clone().into(),
350
						maybe_periodic: Some((4u64, 20)),
351
						origin: signed(i),
352
						_phantom: PhantomData::<u64>::default(),
353
					}),
354
					Some(ScheduledV3Of::<Test> {
355
						maybe_id: Some(vec![255 - i as u8; 320]),
356
						priority: 123,
357
						call: MaybeHashed::Hash(bound_hashed_call.hash()),
358
						maybe_periodic: Some((8u64, 10)),
359
						origin: signed(i),
360
						_phantom: PhantomData::<u64>::default(),
361
					}),
362
					Some(ScheduledV3Of::<Test> {
363
						maybe_id: Some(vec![i as u8; 320]),
364
						priority: 123,
365
						call: MaybeHashed::Hash(undecodable_hash),
366
						maybe_periodic: Some((4u64, 20)),
367
						origin: root(),
368
						_phantom: PhantomData::<u64>::default(),
369
					}),
370
				];
371
				frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old);
372
			}
373

            
374
			let state = v3::MigrateToV4::<Test>::pre_upgrade().unwrap();
375
			let _w = v3::MigrateToV4::<Test>::on_runtime_upgrade();
376
			v3::MigrateToV4::<Test>::post_upgrade(state).unwrap();
377

            
378
			let mut x = Agenda::<Test>::iter().map(|x| (x.0, x.1.into_inner())).collect::<Vec<_>>();
379
			x.sort_by_key(|x| x.0);
380

            
381
			let bound_large_call = Preimage::bound(large_call).unwrap();
382
			assert!(bound_large_call.lookup_needed());
383
			let bound_small_call = Preimage::bound(small_call).unwrap();
384
			assert!(!bound_small_call.lookup_needed());
385

            
386
			let expected = vec![
387
				(
388
					0,
389
					vec![
390
						Some(ScheduledOf::<Test> {
391
							maybe_id: None,
392
							priority: 10,
393
							call: bound_small_call.clone(),
394
							maybe_periodic: None,
395
							origin: root(),
396
							_phantom: PhantomData::<u64>::default(),
397
						}),
398
						None,
399
						Some(ScheduledOf::<Test> {
400
							maybe_id: Some(blake2_256(&[0u8; 32])),
401
							priority: 123,
402
							call: bound_large_call.clone(),
403
							maybe_periodic: Some((4u64, 20)),
404
							origin: signed(0),
405
							_phantom: PhantomData::<u64>::default(),
406
						}),
407
						Some(ScheduledOf::<Test> {
408
							maybe_id: Some(blake2_256(&[255u8; 320])),
409
							priority: 123,
410
							call: Bounded::from_legacy_hash(bound_hashed_call.hash()),
411
							maybe_periodic: Some((8u64, 10)),
412
							origin: signed(0),
413
							_phantom: PhantomData::<u64>::default(),
414
						}),
415
						None,
416
					],
417
				),
418
				(
419
					1,
420
					vec![
421
						Some(ScheduledOf::<Test> {
422
							maybe_id: None,
423
							priority: 11,
424
							call: bound_small_call.clone(),
425
							maybe_periodic: None,
426
							origin: root(),
427
							_phantom: PhantomData::<u64>::default(),
428
						}),
429
						None,
430
						Some(ScheduledOf::<Test> {
431
							maybe_id: Some(blake2_256(&[1u8; 32])),
432
							priority: 123,
433
							call: bound_large_call.clone(),
434
							maybe_periodic: Some((4u64, 20)),
435
							origin: signed(1),
436
							_phantom: PhantomData::<u64>::default(),
437
						}),
438
						Some(ScheduledOf::<Test> {
439
							maybe_id: Some(blake2_256(&[254u8; 320])),
440
							priority: 123,
441
							call: Bounded::from_legacy_hash(bound_hashed_call.hash()),
442
							maybe_periodic: Some((8u64, 10)),
443
							origin: signed(1),
444
							_phantom: PhantomData::<u64>::default(),
445
						}),
446
						None,
447
					],
448
				),
449
			];
450
			for (outer, (i, j)) in x.iter().zip(expected.iter()).enumerate() {
451
				assert_eq!(i.0, j.0);
452
				for (inner, (x, y)) in i.1.iter().zip(j.1.iter()).enumerate() {
453
					assert_eq!(x, y, "at index: outer {} inner {}", outer, inner);
454
				}
455
			}
456
			assert_eq_uvec!(x, expected);
457

            
458
			assert_eq!(StorageVersion::get::<Scheduler>(), 4);
459
		});
460
	}
461

            
462
	#[test]
463
	#[allow(deprecated)]
464
	fn migration_v3_to_v4_too_large_calls_are_ignored() {
465
		new_test_ext().execute_with(|| {
466
			// Assume that we are at V3.
467
			StorageVersion::new(3).put::<Scheduler>();
468

            
469
			let too_large_call = RuntimeCall::System(frame_system::Call::remark {
470
				remark: vec![0; <Test as Config>::Preimages::MAX_LENGTH + 1],
471
			});
472

            
473
			let i = 0u64;
474
			let k = i.twox_64_concat();
475
			let old = vec![Some(ScheduledV3Of::<Test> {
476
				maybe_id: None,
477
				priority: 1,
478
				call: too_large_call.clone().into(),
479
				maybe_periodic: None,
480
				origin: root(),
481
				_phantom: PhantomData::<u64>::default(),
482
			})];
483
			frame_support::migration::put_storage_value(b"Scheduler", b"Agenda", &k, old);
484

            
485
			// The pre_upgrade hook fails:
486
			let err = v3::MigrateToV4::<Test>::pre_upgrade().unwrap_err();
487
			assert_eq!(DispatchError::from("Call is too large."), err);
488
			// But the migration itself works:
489
			let _w = v3::MigrateToV4::<Test>::on_runtime_upgrade();
490

            
491
			let mut x = Agenda::<Test>::iter().map(|x| (x.0, x.1.into_inner())).collect::<Vec<_>>();
492
			x.sort_by_key(|x| x.0);
493
			// The call becomes `None`.
494
			let expected = vec![(0, vec![None])];
495
			assert_eq_uvec!(x, expected);
496

            
497
			assert_eq!(StorageVersion::get::<Scheduler>(), 4);
498
		});
499
	}
500

            
501
	#[test]
502
	fn cleanup_agendas_works() {
503
		use sp_core::bounded_vec;
504
		new_test_ext().execute_with(|| {
505
			StorageVersion::new(4).put::<Scheduler>();
506

            
507
			let call = RuntimeCall::System(frame_system::Call::remark { remark: vec![] });
508
			let bounded_call = Preimage::bound(call).unwrap();
509
			let some = Some(ScheduledOf::<Test> {
510
				maybe_id: None,
511
				priority: 1,
512
				call: bounded_call,
513
				maybe_periodic: None,
514
				origin: root(),
515
				_phantom: Default::default(),
516
			});
517

            
518
			// Put some empty, and some non-empty agendas in there.
519
			let test_data: Vec<(
520
				BoundedVec<Option<ScheduledOf<Test>>, <Test as Config>::MaxScheduledPerBlock>,
521
				Option<
522
					BoundedVec<Option<ScheduledOf<Test>>, <Test as Config>::MaxScheduledPerBlock>,
523
				>,
524
			)> = vec![
525
				(bounded_vec![some.clone()], Some(bounded_vec![some.clone()])),
526
				(bounded_vec![None, some.clone()], Some(bounded_vec![None, some.clone()])),
527
				(bounded_vec![None, some.clone(), None], Some(bounded_vec![None, some.clone()])),
528
				(bounded_vec![some.clone(), None, None], Some(bounded_vec![some.clone()])),
529
				(bounded_vec![None, None], None),
530
				(bounded_vec![None, None, None], None),
531
				(bounded_vec![], None),
532
			];
533

            
534
			// Insert all the agendas.
535
			for (i, test) in test_data.iter().enumerate() {
536
				Agenda::<Test>::insert(i as u64, test.0.clone());
537
			}
538

            
539
			// Run the migration.
540
			let data = v4::CleanupAgendas::<Test>::pre_upgrade().unwrap();
541
			let _w = v4::CleanupAgendas::<Test>::on_runtime_upgrade();
542
			v4::CleanupAgendas::<Test>::post_upgrade(data).unwrap();
543

            
544
			// Check that the post-state is correct.
545
			for (i, test) in test_data.iter().enumerate() {
546
				match test.1.clone() {
547
					None => assert!(
548
						!Agenda::<Test>::contains_key(i as u64),
549
						"Agenda {} should be removed",
550
						i
551
					),
552
					Some(new) => {
553
						assert_eq!(Agenda::<Test>::get(i as u64), new, "Agenda wrong {}", i)
554
					},
555
				}
556
			}
557
		});
558
	}
559

            
560
	fn signed(i: u64) -> OriginCaller {
561
		system::RawOrigin::Signed(i).into()
562
	}
563
}